[Swift-commit] r7092 - in SwiftApps/Swift-MapRed/mapred_chirp_combiner: . old_exec_scripts
yadunandb at ci.uchicago.edu
yadunandb at ci.uchicago.edu
Fri Sep 20 16:38:32 CDT 2013
Author: yadunandb
Date: 2013-09-20 16:38:32 -0500 (Fri, 20 Sep 2013)
New Revision: 7092
Added:
SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/
SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/client.sh
SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/parcombiner_nc.sh
SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/server.sh
SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/teragen_wrap_nc.sh
Removed:
SwiftApps/Swift-MapRed/mapred_chirp_combiner/client.sh
SwiftApps/Swift-MapRed/mapred_chirp_combiner/parcombiner_nc.sh
SwiftApps/Swift-MapRed/mapred_chirp_combiner/server.sh
SwiftApps/Swift-MapRed/mapred_chirp_combiner/teragen_wrap_nc.sh
SwiftApps/Swift-MapRed/mapred_chirp_combiner/test
Modified:
SwiftApps/Swift-MapRed/mapred_chirp_combiner/LAB_RECORDS
SwiftApps/Swift-MapRed/mapred_chirp_combiner/RESULT_FILE
SwiftApps/Swift-MapRed/mapred_chirp_combiner/client.c
SwiftApps/Swift-MapRed/mapred_chirp_combiner/compile
SwiftApps/Swift-MapRed/mapred_chirp_combiner/err_file
SwiftApps/Swift-MapRed/mapred_chirp_combiner/final_result
SwiftApps/Swift-MapRed/mapred_chirp_combiner/parcombiner_c_exec.sh
SwiftApps/Swift-MapRed/mapred_chirp_combiner/run.sh
SwiftApps/Swift-MapRed/mapred_chirp_combiner/server.c
SwiftApps/Swift-MapRed/mapred_chirp_combiner/teragen_wrap_c_exec.sh
Log:
server.c and client.c now act as the standalone remote script executors.
Tested, and works on MCS systems
Modified: SwiftApps/Swift-MapRed/mapred_chirp_combiner/LAB_RECORDS
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/LAB_RECORDS 2013-09-20 02:43:24 UTC (rev 7091)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/LAB_RECORDS 2013-09-20 21:38:32 UTC (rev 7092)
@@ -1180,3 +1180,598 @@
parcombiner_c_exec.sh: line 26: 24727 Segmentation fault (core dumped) ./exec_client $SITE $SITE.$ID.sh &>$SITE.$ID.txt
---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1140-t7fmd7k7
+Progress: time: Thu, 19 Sep 2013 11:40:51 -0500
+Progress: time: Thu, 19 Sep 2013 11:40:56 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 11:41:00 -0500 Submitted:1 Finished successfully:2
+Progress: time: Thu, 19 Sep 2013 11:41:04 -0500 Submitted:1 Finished successfully:3
+Final status: Thu, 19 Sep 2013 11:41:05 -0500 Finished successfully:4
+real 17.77
+user 5.72
+sys 1.00
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT : 0
+EXPECTED COUNT: 300000
+cp: cannot create regular file `./exec_client': File exists
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:10 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1526-369id10c
+Progress: time: Thu, 19 Sep 2013 15:26:11 -0500
+Progress: time: Thu, 19 Sep 2013 15:26:17 -0500 Submitting:9 Submitted:1
+Progress: time: Thu, 19 Sep 2013 15:26:19 -0500 Submitting:8 Submitted:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 15:26:22 -0500 Submitting:6 Submitted:1 Finished successfully:3
+Progress: time: Thu, 19 Sep 2013 15:26:25 -0500 Submitting:4 Submitted:1 Finished successfully:5
+Progress: time: Thu, 19 Sep 2013 15:26:26 -0500 Submitting:2 Submitted:1 Finished successfully:7
+Progress: time: Thu, 19 Sep 2013 15:26:29 -0500 Submitting:1 Submitted:1 Finished successfully:8
+Progress: time: Thu, 19 Sep 2013 15:26:32 -0500 Submitted:1 Finished successfully:9
+Progress: time: Thu, 19 Sep 2013 15:26:36 -0500 Submitted:1 Finished successfully:10
+Execution failed:
+ Exception in bash:
+ Arguments: [parcombiner_c_exec.sh, tgen0000.out tgen0001.out tgen0002.out tgen0003.out tgen0004.out tgen0005.out tgen0006.out tgen0007.out tgen0008.out tgen0009.out]
+ Host: churn
+ Directory: teragen-20130919-1526-369id10c/jobs/1/bash-1786khfl
+
+Caused by:
+ Error staging in file: /nfs2/yadunand/Swift-MapRed/mapred_chirp_combiner/./parcombiner_c_exec.sh (No such file or directory)
+org.globus.cog.abstraction.impl.common.execution.JobException: Error staging in file: /nfs2/yadunand/Swift-MapRed/mapred_chirp_combiner/./parcombiner_c_exec.sh (No such file or directory) (exit code: 520)
+ at org.globus.cog.abstraction.coaster.service.local.JobStatusHandler.requestComplete(JobStatusHandler.java:38)
+ at org.globus.cog.karajan.workflow.service.handlers.RequestHandler.receiveCompleted(RequestHandler.java:88)
+ at org.globus.cog.karajan.workflow.service.channels.AbstractKarajanChannel.handleRequest(AbstractKarajanChannel.java:518)
+ at org.globus.cog.karajan.workflow.service.channels.AbstractStreamKarajanChannel.stepNIO(AbstractStreamKarajanChannel.java:238)
+ at org.globus.cog.karajan.workflow.service.channels.NIOMultiplexer.loop(NIOMultiplexer.java:97)
+ at org.globus.cog.karajan.workflow.service.channels.NIOMultiplexer.run(NIOMultiplexer.java:56)
+ (exit code: 520)
+ comb_data, teragen.swift, line 33
+real 28.42
+user 10.06
+sys 2.49
+chunks 10 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT : 0
+EXPECTED COUNT: 1000000
+cp: cannot create regular file `./exec_client': File exists
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:10 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1527-58fusym5
+Progress: time: Thu, 19 Sep 2013 15:27:19 -0500
+Progress: time: Thu, 19 Sep 2013 15:27:24 -0500 Submitting:9 Submitted:1
+Progress: time: Thu, 19 Sep 2013 15:27:26 -0500 Submitting:8 Submitted:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 15:27:29 -0500 Submitting:7 Submitted:1 Finished successfully:2
+Progress: time: Thu, 19 Sep 2013 15:27:31 -0500 Submitting:6 Submitted:1 Finished successfully:3
+Progress: time: Thu, 19 Sep 2013 15:27:33 -0500 Submitting:2 Submitted:1 Finished successfully:7
+Progress: time: Thu, 19 Sep 2013 15:27:36 -0500 Submitting:1 Submitted:1 Finished successfully:8
+Progress: time: Thu, 19 Sep 2013 15:27:38 -0500 Submitted:1 Finished successfully:9
+Progress: time: Thu, 19 Sep 2013 15:27:42 -0500 Submitted:1 Finished successfully:10
+Progress: time: Thu, 19 Sep 2013 15:27:43 -0500 Active:1 Finished successfully:10
+Final status: Thu, 19 Sep 2013 15:27:43 -0500 Finished successfully:11
+real 26.74
+user 9.45
+sys 2.21
+chunks 10 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 1000000
+cp: cp: cannot create regular file `./exec_client'cp: : File existscannot create regular file `./exec_client'
+cannot create regular file `./exec_client': File exists
+cp: : File exists
+cannot create regular file `./exec_client'cp: : File existscannot create regular file `./exec_client'
+: File exists
+cp: cannot create regular file `./exec_client': File exists
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:10 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1541-r3usm7q8
+Progress: time: Thu, 19 Sep 2013 15:41:04 -0500
+Progress: time: Thu, 19 Sep 2013 15:41:10 -0500 Submitting:9 Submitted:1
+Progress: time: Thu, 19 Sep 2013 15:41:13 -0500 Submitting:8 Submitted:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 15:41:16 -0500 Submitting:6 Submitted:1 Finished successfully:3
+Progress: time: Thu, 19 Sep 2013 15:41:19 -0500 Submitting:5 Submitted:1 Finished successfully:4
+Progress: time: Thu, 19 Sep 2013 15:41:24 -0500 Submitting:4 Submitted:1 Finished successfully:5
+Progress: time: Thu, 19 Sep 2013 15:41:27 -0500 Submitting:1 Submitted:1 Finished successfully:8
+Progress: time: Thu, 19 Sep 2013 15:41:31 -0500 Submitted:1 Finished successfully:9
+Progress: time: Thu, 19 Sep 2013 15:41:34 -0500 Submitting:1 Finished successfully:10
+Progress: time: Thu, 19 Sep 2013 15:41:37 -0500 Submitted:1 Finished successfully:10
+Progress: time: Thu, 19 Sep 2013 15:41:38 -0500 Active:1 Finished successfully:10
+Final status: Thu, 19 Sep 2013 15:41:38 -0500 Finished successfully:11
+real 37.08
+user 9.96
+sys 2.40
+chunks 10 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 1000000
+cp: cannot create regular file `./exec_client': File exists
+cp: cannot create regular file `./exec_client': File exists
+cp: cannot create regular file `./exec_client': File existscp:
+cannot create regular file `./exec_client': File exists
+cp: cannot create regular file `./exec_client': File exists
+cp: cannot create regular file `./exec_client': File exists
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1543-k1zhfgug
+Progress: time: Thu, 19 Sep 2013 15:43:22 -0500
+Progress: time: Thu, 19 Sep 2013 15:43:29 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 15:43:30 -0500 Submitting:1 Stage out:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 15:43:32 -0500 Submitted:1 Finished successfully:2
+Progress: time: Thu, 19 Sep 2013 15:43:36 -0500 Submitted:1 Finished successfully:3
+Final status: Thu, 19 Sep 2013 15:43:37 -0500 Finished successfully:4
+real 17.15
+user 5.94
+sys 1.09
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 300000
+cp: cannot create regular file `./exec_client': File exists
+cp: cannot create regular file `./exec_client': File exists
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1546-rxfdjnuf
+Progress: time: Thu, 19 Sep 2013 15:46:10 -0500
+Progress: time: Thu, 19 Sep 2013 15:46:15 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 15:46:17 -0500 Submitting:1 Submitted:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 15:46:19 -0500 Submitted:1 Finished successfully:2
+Progress: time: Thu, 19 Sep 2013 15:46:23 -0500 Submitted:1 Finished successfully:3
+Final status: Thu, 19 Sep 2013 15:46:24 -0500 Finished successfully:4
+real 16.15
+user 5.72
+sys 1.16
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 300000
+cat: churn.17035.txt: No such file or directory
+cat: thwomp.14738.txt: No such file or directory
+cat: trounce.14436.txt: No such file or directory
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1551-c0n6032f
+Progress: time: Thu, 19 Sep 2013 15:51:29 -0500
+Progress: time: Thu, 19 Sep 2013 15:51:35 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 15:51:36 -0500 Submitting:1 Stage out:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 15:51:38 -0500 Submitted:1 Finished successfully:2
+Progress: time: Thu, 19 Sep 2013 15:51:39 -0500 Finished successfully:4
+Final status: Thu, 19 Sep 2013 15:51:39 -0500 Finished successfully:4
+real 12.37
+user 5.56
+sys 0.98
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 300000
+cat: crush.8560.txt: No such file or directory
+cat: grind.31695.txt: No such file or directory
+cat: stomp.21449.txt: No such file or directory
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1556-pb5kt7d7
+Progress: time: Thu, 19 Sep 2013 15:56:45 -0500
+Progress: time: Thu, 19 Sep 2013 15:56:50 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 15:56:54 -0500 Submitted:1 Finished successfully:2
+Progress: time: Thu, 19 Sep 2013 15:56:59 -0500 Submitted:1 Finished successfully:3
+Final status: Thu, 19 Sep 2013 15:56:59 -0500 Finished successfully:4
+real 16.63
+user 5.70
+sys 0.95
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 300000
+cat: thrash.13566.txt: No such file or directory
+cat: thwomp.10347.txt: No such file or directory
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1609-aki2sm14
+Progress: time: Thu, 19 Sep 2013 16:09:52 -0500
+Progress: time: Thu, 19 Sep 2013 16:09:58 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 16:10:02 -0500 Submitting:1 Submitted:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 16:10:03 -0500 Submitting:1 Finished successfully:2
+Progress: time: Thu, 19 Sep 2013 16:10:09 -0500 Submitted:1 Finished successfully:3
+Final status: Thu, 19 Sep 2013 16:10:09 -0500 Finished successfully:4
+real 19.31
+user 5.92
+sys 1.18
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 300000
+cat: grind.3335.txt: No such file or directory
+cat: thrash.22590.txt: No such file or directory
+cat: vanquish.27168.txt: No such file or directory
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1625-hcf0aol7
+Progress: time: Thu, 19 Sep 2013 16:25:35 -0500
+Progress: time: Thu, 19 Sep 2013 16:25:40 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 16:25:42 -0500 Submitting:1 Submitted:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 16:25:46 -0500 Submitted:1 Finished successfully:2
+Progress: time: Thu, 19 Sep 2013 16:25:52 -0500 Submitted:1 Finished successfully:3
+Final status: Thu, 19 Sep 2013 16:25:52 -0500 Finished successfully:4
+real 19.55
+user 6.05
+sys 1.26
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 300000
+cat: churn.29488.txt: No such file or directory
+cat: grind.32469.txt: No such file or directory
+cat: stomp.4237.txt: No such file or directory
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1626-ifqcvmbe
+Progress: time: Thu, 19 Sep 2013 16:26:55 -0500
+Progress: time: Thu, 19 Sep 2013 16:27:00 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 16:27:04 -0500 Submitting:1 Submitted:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 16:27:05 -0500 Submitting:1 Stage out:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 16:27:06 -0500 Stage out:1 Finished successfully:2
+Execution failed:
+ Exception in bash:
+ Arguments: [parcombiner_c_exec.sh, tgen0000.out tgen0001.out tgen0002.out]
+ Host: steamroller
+ Directory: teragen-20130919-1626-ifqcvmbe/jobs/8/bash-8a8lmhfl
+
+Caused by:
+ Application /bin/bash failed with an exit code of 2
+ comb_data, teragen.swift, line 33
+real 13.36
+user 5.54
+sys 0.96
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 300000
+cat: churn.29488.txt: No such file or directory
+cat: grind.32469.txt: No such file or directory
+cat: stomp.4237.txt: No such file or directory
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1642-yayzuxq5
+Progress: time: Thu, 19 Sep 2013 16:42:29 -0500
+Progress: time: Thu, 19 Sep 2013 16:42:36 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 16:42:38 -0500 Stage in:1 Finished successfully:2
+Progress: time: Thu, 19 Sep 2013 16:42:43 -0500 Submitted:1 Finished successfully:3
+Execution failed:
+ Exception in bash:
+ Arguments: [parcombiner_c_exec.sh, tgen0000.out tgen0001.out tgen0002.out]
+ Host: thwomp
+ Directory: teragen-20130919-1642-yayzuxq5/jobs/a/bash-an08nhfl
+
+Caused by:
+ Application /bin/bash failed with an exit code of 2
+ comb_data, teragen.swift, line 33
+real 16.38
+user 5.81
+sys 0.89
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 300000
+cat: churn.29488.txt: No such file or directory
+cat: grind.32469.txt: No such file or directory
+cat: stomp.4237.txt: No such file or directory
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1643-x97al4sa
+Progress: time: Thu, 19 Sep 2013 16:43:41 -0500
+Progress: time: Thu, 19 Sep 2013 16:43:46 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 16:43:48 -0500 Submitting:1 Submitted:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 16:43:50 -0500 Submitted:1 Finished successfully:2
+Progress: time: Thu, 19 Sep 2013 16:43:57 -0500 Submitted:1 Finished successfully:3
+Execution failed:
+ Exception in bash:
+ Arguments: [parcombiner_c_exec.sh, tgen0000.out tgen0001.out tgen0002.out]
+ Host: thrash
+ Directory: teragen-20130919-1643-x97al4sa/jobs/c/bash-c7o9nhfl
+
+Caused by:
+ Application /bin/bash failed with an exit code of 2
+ comb_data, teragen.swift, line 33
+real 18.40
+user 5.98
+sys 1.28
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT :
+EXPECTED COUNT: 300000
+cat: churn.29488.txt: No such file or directory
+cat: grind.32469.txt: No such file or directory
+cat: stomp.4237.txt: No such file or directory
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1645-vl31exq8
+Progress: time: Thu, 19 Sep 2013 16:45:14 -0500
+Progress: time: Thu, 19 Sep 2013 16:45:21 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 16:45:22 -0500 Submitting:1 Submitted:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 16:45:23 -0500 Stage out:1 Finished successfully:3
+Final status: Thu, 19 Sep 2013 16:45:23 -0500 Finished successfully:4
+real 10.91
+user 5.49
+sys 0.65
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT : 0
+EXPECTED COUNT: 300000
+cat: churn.29488.txt: No such file or directory
+cat: grind.32469.txt: No such file or directory
+cat: stomp.4237.txt: No such file or directory
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1705-vha2ynx1
+Progress: time: Thu, 19 Sep 2013 17:05:13 -0500
+Progress: time: Thu, 19 Sep 2013 17:05:20 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 17:05:21 -0500 Submitting:1 Submitted:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 17:05:23 -0500 Submitted:1 Finished successfully:2
+Progress: time: Thu, 19 Sep 2013 17:05:24 -0500 Finished successfully:4
+Final status: Thu, 19 Sep 2013 17:05:24 -0500 Finished successfully:4
+real 12.63
+user 5.57
+sys 0.94
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT : 0
+EXPECTED COUNT: 300000
+cat: churn.29488.txt: No such file or directory
+cat: grind.32469.txt: No such file or directory
+cat: stomp.4237.txt: No such file or directory
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1711-cuacgapb
+Progress: time: Thu, 19 Sep 2013 17:11:44 -0500
+Progress: time: Thu, 19 Sep 2013 17:11:49 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 17:11:53 -0500 Submitting:1 Submitted:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 17:11:58 -0500 Submitted:1 Finished successfully:3
+Final status: Thu, 19 Sep 2013 17:11:59 -0500 Finished successfully:4
+real 17.20
+user 5.39
+sys 0.96
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT : 0
+EXPECTED COUNT: 300000
+cat: churn.29488.txt: No such file or directory
+cat: grind.32469.txt: No such file or directory
+cat: stomp.4237.txt: No such file or directory
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1721-d68jzzp4
+Progress: time: Thu, 19 Sep 2013 17:21:08 -0500
+Progress: time: Thu, 19 Sep 2013 17:21:14 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 17:21:16 -0500 Submitting:1 Submitted:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 17:21:18 -0500 Submitted:1 Finished successfully:2
+Progress: time: Thu, 19 Sep 2013 17:21:23 -0500 Submitted:1 Finished successfully:3
+Final status: Thu, 19 Sep 2013 17:21:23 -0500 Finished successfully:4
+real 17.01
+user 6.08
+sys 1.23
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT : 0
+EXPECTED COUNT: 300000
+cat: churn.29488.txt: No such file or directory
+cat: grind.32469.txt: No such file or directory
+cat: stomp.4237.txt: No such file or directory
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1723-1kyci1m9
+Progress: time: Thu, 19 Sep 2013 17:23:31 -0500
+Progress: time: Thu, 19 Sep 2013 17:23:35 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 17:23:37 -0500 Submitting:1 Submitted:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 17:23:40 -0500 Submitted:1 Finished successfully:2
+Progress: time: Thu, 19 Sep 2013 17:23:45 -0500 Submitted:1 Finished successfully:3
+Final status: Thu, 19 Sep 2013 17:23:46 -0500 Finished successfully:4
+real 17.04
+user 6.08
+sys 1.15
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT : 0
+EXPECTED COUNT: 300000
+cat: churn.29488.txt: No such file or directory
+cat: grind.32469.txt: No such file or directory
+cat: stomp.4237.txt: No such file or directory
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1725-d7udf57a
+Progress: time: Thu, 19 Sep 2013 17:25:13 -0500
+Progress: time: Thu, 19 Sep 2013 17:25:18 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 17:25:20 -0500 Submitting:1 Submitted:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 17:25:22 -0500 Submitted:1 Finished successfully:2
+Progress: time: Thu, 19 Sep 2013 17:25:23 -0500 Stage out:1 Finished successfully:3
+Final status: Thu, 19 Sep 2013 17:25:23 -0500 Finished successfully:4
+real 12.07
+user 5.65
+sys 1.00
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT : 0
+EXPECTED COUNT: 300000
+cat: churn.29488.txt: No such file or directory
+cat: grind.32469.txt: No such file or directory
+cat: stomp.4237.txt: No such file or directory
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1728-vdph5ji3
+Progress: time: Thu, 19 Sep 2013 17:28:07 -0500
+Progress: time: Thu, 19 Sep 2013 17:28:13 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 17:28:14 -0500 Submitting:1 Submitted:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 17:28:16 -0500 Submitted:1 Finished successfully:2
+Progress: time: Thu, 19 Sep 2013 17:28:22 -0500 Submitted:1 Finished successfully:3
+Final status: Thu, 19 Sep 2013 17:28:23 -0500 Finished successfully:4
+real 18.78
+user 7.11
+sys 1.27
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT : 0
+EXPECTED COUNT: 300000
+cat: churn.29488.txt: No such file or directory
+cat: grind.32469.txt: No such file or directory
+cat: stomp.4237.txt: No such file or directory
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1733-7g9h36mc
+Progress: time: Thu, 19 Sep 2013 17:33:14 -0500
+Progress: time: Thu, 19 Sep 2013 17:33:18 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 17:33:23 -0500 Submitted:1 Finished successfully:2
+Progress: time: Thu, 19 Sep 2013 17:33:28 -0500 Submitted:1 Finished successfully:3
+Final status: Thu, 19 Sep 2013 17:33:29 -0500 Finished successfully:4
+real 17.43
+user 5.60
+sys 1.06
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT : 0
+EXPECTED COUNT: 300000
+cat: churn.29488.txt: No such file or directory
+cat: grind.32469.txt: No such file or directory
+cat: stomp.4237.txt: No such file or directory
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1744-y51ic8y8
+Progress: time: Thu, 19 Sep 2013 17:44:24 -0500
+Progress: time: Thu, 19 Sep 2013 17:44:30 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 17:44:31 -0500 Submitting:1 Submitted:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 17:44:33 -0500 Submitted:1 Finished successfully:2
+Progress: time: Thu, 19 Sep 2013 17:44:34 -0500 Stage out:1 Finished successfully:3
+Final status: Thu, 19 Sep 2013 17:44:35 -0500 Finished successfully:4
+real 12.63
+user 5.88
+sys 0.91
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 300000
+EXPECTED COUNT: 300000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:3 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1749-n9lybheg
+Progress: time: Thu, 19 Sep 2013 17:49:05 -0500
+Progress: time: Thu, 19 Sep 2013 17:49:09 -0500 Submitting:2 Submitted:1
+Progress: time: Thu, 19 Sep 2013 17:49:12 -0500 Submitted:1 Finished successfully:2
+Progress: time: Thu, 19 Sep 2013 17:49:19 -0500 Submitted:1 Finished successfully:3
+Final status: Thu, 19 Sep 2013 17:49:20 -0500 Finished successfully:4
+real 16.75
+user 5.72
+sys 0.88
+chunks 3 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 300000
+EXPECTED COUNT: 300000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (With local combiners), SITE:anl, CHUNKS:10 X FILESIZE:10
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130919-1750-mhqa6idg
+Progress: time: Thu, 19 Sep 2013 17:50:23 -0500
+Progress: time: Thu, 19 Sep 2013 17:50:29 -0500 Submitting:9 Submitted:1
+Progress: time: Thu, 19 Sep 2013 17:50:31 -0500 Submitting:8 Submitted:1 Finished successfully:1
+Progress: time: Thu, 19 Sep 2013 17:50:33 -0500 Submitting:6 Submitted:1 Finished successfully:3
+Progress: time: Thu, 19 Sep 2013 17:50:37 -0500 Submitting:3 Submitted:1 Finished successfully:6
+Progress: time: Thu, 19 Sep 2013 17:50:38 -0500 Submitting:2 Submitted:1 Finished successfully:7
+Progress: time: Thu, 19 Sep 2013 17:50:40 -0500 Submitted:1 Finished successfully:9
+Progress: time: Thu, 19 Sep 2013 17:50:45 -0500 Submitted:1 Finished successfully:10
+Progress: time: Thu, 19 Sep 2013 17:50:46 -0500 Active:1 Finished successfully:10
+Final status: Thu, 19 Sep 2013 17:50:47 -0500 Finished successfully:11
+real 25.84
+user 7.50
+sys 2.08
+chunks 10 filesize 10 site anl type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 1000000
+EXPECTED COUNT: 1000000
+---------------------------------------------
Modified: SwiftApps/Swift-MapRed/mapred_chirp_combiner/RESULT_FILE
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/RESULT_FILE 2013-09-20 02:43:24 UTC (rev 7091)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/RESULT_FILE 2013-09-20 21:38:32 UTC (rev 7092)
@@ -1 +0,0 @@
-steamroller /sandbox/yadunand/input_30326.txt
Modified: SwiftApps/Swift-MapRed/mapred_chirp_combiner/client.c
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/client.c 2013-09-20 02:43:24 UTC (rev 7091)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/client.c 2013-09-20 21:38:32 UTC (rev 7092)
@@ -28,7 +28,6 @@
if (sa->sa_family == AF_INET) {
return &(((struct sockaddr_in*)sa)->sin_addr);
}
-
return &(((struct sockaddr_in6*)sa)->sin6_addr);
}
@@ -53,7 +52,8 @@
memset(&hints, 0, sizeof hints);
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
-
+
+ // Leaving PORT as NULL
if ((rv = getaddrinfo(argv[1], PORT, &hints, &servinfo)) != 0) {
fprintf(stderr, "getaddrinfo: %s\n", gai_strerror(rv));
return 1;
@@ -83,12 +83,12 @@
inet_ntop(p->ai_family, get_in_addr((struct sockaddr *)p->ai_addr),
s, sizeof s);
- //printf("client: connecting to %s\n", s);
+ printf("client: connecting to %s\n", s);
freeaddrinfo(servinfo); // all done with this structure
//Send message to server
- //printf("Client sending contents of file %s to server \n",argv[1]);
+ printf("Client sending contents of file %s to server \n",argv[2]);
// Read file contents into a buffer;
if ( (f_in = fopen(argv[2],"r")) == NULL ){
@@ -104,10 +104,9 @@
buffer = (char *) malloc (file_size);
fread(buffer, file_size, 1, f_in);
buffer[file_size] = '\0';
- //printf("String : %s\n", buffer);
fclose(f_in);
- //printf("Content of file : \n%s\n", buffer);
+ printf("Content of file : \n%s\n", buffer);
if (send(sockfd, buffer, file_size, 0) == -1)
perror("send");
@@ -115,12 +114,12 @@
//Receive message back from server
reply_buf = (char *)malloc(MAXDATASIZE);
- //printf("Client receiving message from server \n");
+ printf("Client receiving message from server \n");
if ((numbytes = recv(sockfd, reply_buf, MAXDATASIZE-1, 0)) == -1) {
perror("recv");
exit(1);
}
-
+ printf("Numbytes recv'd : %d\n", numbytes);
reply_buf[numbytes] = '\0';
printf("%s",reply_buf);
free(reply_buf);
Deleted: SwiftApps/Swift-MapRed/mapred_chirp_combiner/client.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/client.sh 2013-09-20 02:43:24 UTC (rev 7091)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/client.sh 2013-09-20 21:38:32 UTC (rev 7092)
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-REMOTE_PORT=20033
-
-if [ "$1" == "" ]
-then
- echo "Client needs an arg for remote"
- exit 0
-fi
-
-EXEC_FILE="tmp.$RANDOM"
-
-PORT=$(($REMOTE_PORT + 10))
-echo "$(hostname -f) $PORT" > $EXEC_FILE
-cat <<'EOF' >> $EXEC_FILE
-#!/bin/bash
-echo "Recombiner on $HOSTNAME"
-echo $(hostname -f)
-EOF
-
-listen_for_reply()
-{
- nc -l $PORT > reply;
- echo "Connection received back on $PORT"
-}
-
-listen_for_reply &
-nc $1 $REMOTE_PORT < $EXEC_FILE
-echo "nc $1 $PORT;"
-
Modified: SwiftApps/Swift-MapRed/mapred_chirp_combiner/compile
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/compile 2013-09-20 02:43:24 UTC (rev 7091)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/compile 2013-09-20 21:38:32 UTC (rev 7092)
@@ -5,3 +5,6 @@
chmod a+x exec_server
chmod a+x exec_client
+
+cp exec_server ~/bin/exec_server
+cp exec_client ~/bin/exec_client
Modified: SwiftApps/Swift-MapRed/mapred_chirp_combiner/err_file
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/err_file 2013-09-20 02:43:24 UTC (rev 7091)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/err_file 2013-09-20 21:38:32 UTC (rev 7092)
@@ -1,3 +1,3 @@
-cp: cp: cannot create regular file `./exec_client'cannot create regular file `./exec_client': Text file busy: Text file busy
-
-parcombiner_c_exec.sh: line 26: 24727 Segmentation fault (core dumped) ./exec_client $SITE $SITE.$ID.sh &>$SITE.$ID.txt
+cat: churn.29488.txt: No such file or directory
+cat: grind.32469.txt: No such file or directory
+cat: stomp.4237.txt: No such file or directory
Modified: SwiftApps/Swift-MapRed/mapred_chirp_combiner/final_result
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/final_result 2013-09-20 02:43:24 UTC (rev 7091)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/final_result 2013-09-20 21:38:32 UTC (rev 7092)
@@ -1,5 +1,123 @@
-Content of stomp.20006.txt is
-Content of vanquish.11894.txt is
-Content of thrash.9680.txt is
-Final results (on stomp):
-
+parcombiner_c_exec running at /sandbox/yadunand/swiftwork/teragen-20130919-1750-mhqa6idg/jobs/7/bash-7z0yphfl
+client: connecting to 140.221.8.75
+Client sending contents of file churn.28341.sh to server
+Content of file :
+#!/bin/bash
+ID=28341
+FILES=(/sandbox/yadunand/input_5455.txt)
+
+SUM=0;COUNT=0;
+for file in ${FILES[*]}; do
+ [ ! -f $file ] && echo "$file is missing" >> $HOSTNAME.parrot_run.log
+ RES=($(awk '{ sum += $1 } END { print sum,NR }' $file))
+ SUM=$(($SUM+${RES[0]}))
+ COUNT=$(($COUNT+${RES[1]}))
+ rm $file &
+done;
+echo -e "$SUM $COUNT"
+
+Client receiving message from server
+Numbytes recv'd : 22
+49985593630853 100000
+client: connecting to 140.221.8.80
+Client sending contents of file trounce.15296.sh to server
+Content of file :
+#!/bin/bash
+ID=15296
+FILES=(/sandbox/yadunand/input_5129.txt)
+
+SUM=0;COUNT=0;
+for file in ${FILES[*]}; do
+ [ ! -f $file ] && echo "$file is missing" >> $HOSTNAME.parrot_run.log
+ RES=($(awk '{ sum += $1 } END { print sum,NR }' $file))
+ SUM=$(($SUM+${RES[0]}))
+ COUNT=$(($COUNT+${RES[1]}))
+ rm $file &
+done;
+echo -e "$SUM $COUNT"
+
+Client receiving message from server
+Numbytes recv'd : 22
+50039771820812 100000
+client: connecting to 140.221.8.74
+Client sending contents of file grind.14953.sh to server
+Content of file :
+#!/bin/bash
+ID=14953
+FILES=(/sandbox/yadunand/input_22065.txt)
+
+SUM=0;COUNT=0;
+for file in ${FILES[*]}; do
+ [ ! -f $file ] && echo "$file is missing" >> $HOSTNAME.parrot_run.log
+ RES=($(awk '{ sum += $1 } END { print sum,NR }' $file))
+ SUM=$(($SUM+${RES[0]}))
+ COUNT=$(($COUNT+${RES[1]}))
+ rm $file &
+done;
+echo -e "$SUM $COUNT"
+
+Client receiving message from server
+Numbytes recv'd : 22
+49935537269888 100000
+client: connecting to 140.221.8.73
+Client sending contents of file steamroller.32624.sh to server
+Content of file :
+#!/bin/bash
+ID=32624
+FILES=(/sandbox/yadunand/input_13617.txt /sandbox/yadunand/input_30290.txt)
+
+SUM=0;COUNT=0;
+for file in ${FILES[*]}; do
+ [ ! -f $file ] && echo "$file is missing" >> $HOSTNAME.parrot_run.log
+ RES=($(awk '{ sum += $1 } END { print sum,NR }' $file))
+ SUM=$(($SUM+${RES[0]}))
+ COUNT=$(($COUNT+${RES[1]}))
+ rm $file &
+done;
+echo -e "$SUM $COUNT"
+
+Client receiving message from server
+Numbytes recv'd : 22
+99923825057728 200000
+client: connecting to 140.221.9.237
+Client sending contents of file stomp.14422.sh to server
+Content of file :
+#!/bin/bash
+ID=14422
+FILES=(/sandbox/yadunand/input_815.txt /sandbox/yadunand/input_7548.txt)
+
+SUM=0;COUNT=0;
+for file in ${FILES[*]}; do
+ [ ! -f $file ] && echo "$file is missing" >> $HOSTNAME.parrot_run.log
+ RES=($(awk '{ sum += $1 } END { print sum,NR }' $file))
+ SUM=$(($SUM+${RES[0]}))
+ COUNT=$(($COUNT+${RES[1]}))
+ rm $file &
+done;
+echo -e "$SUM $COUNT"
+
+Client receiving message from server
+Numbytes recv'd : 22
+99996184007931 200000
+client: connecting to 140.221.8.81
+Client sending contents of file thrash.20240.sh to server
+Content of file :
+#!/bin/bash
+ID=20240
+FILES=(/sandbox/yadunand/input_18840.txt /sandbox/yadunand/input_20497.txt /sandbox/yadunand/input_11594.txt)
+
+SUM=0;COUNT=0;
+for file in ${FILES[*]}; do
+ [ ! -f $file ] && echo "$file is missing" >> $HOSTNAME.parrot_run.log
+ RES=($(awk '{ sum += $1 } END { print sum,NR }' $file))
+ SUM=$(($SUM+${RES[0]}))
+ COUNT=$(($COUNT+${RES[1]}))
+ rm $file &
+done;
+echo -e "$SUM $COUNT"
+
+Client receiving message from server
+Numbytes recv'd : 23
+150220428477325 300000
+Final results (on vanquish):
+500101340264537 1000000
Added: SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/client.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/client.sh (rev 0)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/client.sh 2013-09-20 21:38:32 UTC (rev 7092)
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+REMOTE_PORT=20033
+
+if [ "$1" == "" ]
+then
+ echo "Client needs an arg for remote"
+ exit 0
+fi
+
+EXEC_FILE="tmp.$RANDOM"
+
+PORT=$(($REMOTE_PORT + 10))
+echo "$(hostname -f) $PORT" > $EXEC_FILE
+cat <<'EOF' >> $EXEC_FILE
+#!/bin/bash
+echo "Recombiner on $HOSTNAME"
+echo $(hostname -f)
+EOF
+
+listen_for_reply()
+{
+ nc -l $PORT > reply;
+ echo "Connection received back on $PORT"
+}
+
+listen_for_reply &
+nc $1 $REMOTE_PORT < $EXEC_FILE
+echo "nc $1 $PORT;"
+
Property changes on: SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/client.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/parcombiner_nc.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/parcombiner_nc.sh (rev 0)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/parcombiner_nc.sh 2013-09-20 21:38:32 UTC (rev 7092)
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+FILES=$*
+SUM=0
+COUNT=0
+REMOTE_PORT=29900
+cat $* > RESULT_FILE
+
+# Find the unique sites in the RESULT_FILE to run a combiner there.
+#SITES=($(sort -u -t' ' -k1,1 RESULT_FILE))
+cat <<'EOF' > recombiner.sh
+SUM=0;COUNT=0;
+for file in ${FILES[*]}; do
+ [ ! -f $file ] && echo "$file is missing" >> $HOSTNAME.parrot_run.log
+ RES=($(awk '{ sum += $1 } END { print sum,NR }' $file))
+ SUM=$(($SUM+${RES[0]}))
+ COUNT=$(($COUNT+${RES[1]}))
+ rm $file &
+done;
+echo -e "$SUM $COUNT"
+EOF
+chmod 777 recombiner.sh
+
+
+combine_fetch_process()
+{
+ SITE=$1; shift
+ ID=$1; shift
+ REMOTE_PORT=$1;
+ PORT=$(($REMOTE_PORT+$ID))
+
+ FILES=($(grep $SITE RESULT_FILE | awk '{ print $2 }'))
+ CHIRPDIR=$(dirname ${FILES[0]})
+ #chirp_put ./recombiner.sh $SITE recombiner.sh
+ #ssh $SITE "cd $CHIRPDIR; ./recombiner.sh $ID ${FILES[*]}"
+ #chirp_get $SITE $SITE.$ID.imd $SITE.$ID.txt
+
+ echo -e "$(hostname -f) $PORT\n#!/bin/bash \n ID=$ID \n FILES=(${FILES[*]}) \n" > $SITE.$ID.sh
+ cat recombiner.sh >> $SITE.$ID.sh
+ nc $SITE $REMOTE_PORT < $SITE.$ID.sh
+ nc -l $PORT > $SITE.$ID.txt
+ #rm $SITE.$ID.sh
+}
+
+STAGE2="";
+SITES=($(awk '{ print $1 }' RESULT_FILE | sort -u))
+for SITE in ${SITES[*]}
+do
+ ID=$RANDOM
+ combine_fetch_process $SITE $ID $REMOTE_PORT &
+ STAGE2="$STAGE2 $SITE.$ID.txt"
+done;
+wait
+
+cat ${STAGE2[*]} > TEMP
+SUM=($(awk '{ sum += $1 } END { print sum }' TEMP))
+COUNT=($(awk '{ sum += $2 } END { print sum }' TEMP))
+echo "Final results (on $HOSTNAME):"
+echo "$SUM $COUNT"
+
+exit 0;
\ No newline at end of file
Property changes on: SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/parcombiner_nc.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/server.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/server.sh (rev 0)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/server.sh 2013-09-20 21:38:32 UTC (rev 7092)
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+PORT=20033
+TMP="tmp.$RANDOM.$HOSTNAME"
+nc -l $PORT > $TMP
+chmod +x $TMP
+RETURN_ADDRESS=$(head -n 1 $TMP)
+sed -i -e "1d" $TMP
+./$TMP &> $TMP.reply
+nc $RETURN_ADDRESS < $TMP.reply
Property changes on: SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/server.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/teragen_wrap_nc.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/teragen_wrap_nc.sh (rev 0)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/teragen_wrap_nc.sh 2013-09-20 21:38:32 UTC (rev 7092)
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+
+# By default with ARG1:100 and SLICESIZE=10000, this script will generate
+# 10^6 records.
+ARG1=1
+[ ! -z $1 ] && ARG1=$1
+
+FILE="input_$RANDOM.txt"
+LOWERLIMIT=0
+UPPERLIMIT=1000000000 # 10^9
+SLICESIZE=10000 # 10^4 records padded to 100B would result in 1MB file
+#SLICESIZE=1000 # 10^3 If padded to 100B would result
+
+
+FOLDER="/sandbox/$USER"
+TIMEOUT=600
+NC_LISTEN_PORT=29900
+
+# clean_folder SLEEP_TIME FOLDER_NAME
+clean_folder()
+{
+ sleep $(($1+1));
+ rm $2/* &> /dev/null
+}
+
+exec_server()
+{
+ cd /tmp
+ PORT=$1
+ TMP="/homes/yadunand/tmp.$RANDOM.$HOSTNAME"
+ nc -l $PORT > $TMP
+ chmod +x $TMP
+ RETURN_ADDRESS=$(head -n 1 $TMP)
+ sed -i -e "1d" $TMP
+ ls >> $TMP.reply
+ pwd >> $TMP.reply
+ echo "----" >> $TMP.reply
+ /bin/bash $TMP &> $TMP.reply
+ echo "----" >> $TMP.reply
+ echo "nc $RETURN_ADDRESS < $TMP.reply" >> $TMP.reply
+ nc $RETURN_ADDRESS < $TMP.reply
+}
+
+JOBS_LEVEL=`echo $PWD | grep -o ".*jobs"`
+if [ "$?" == "0" ]
+then
+ mkdir $JOBS_LEVEL/CHIRPING
+ if [ "$?" == "0" ]
+ then
+ killall -u $USER chirp_server;
+ echo "unix:$USER rwlds" > $FOLDER/acl.conf
+ echo "hostname:* rwl" >> $FOLDER/acl.conf
+ timeout $TIMEOUT chirp_server -A $FOLDER/acl.conf -r $FOLDER &
+ exec_server $NC_LISTEN_PORT &
+ fi
+ FILE=$FOLDER/$FILE;
+else
+ FILE=$PWD/$FILE
+fi
+
+shuf -i $LOWERLIMIT-$UPPERLIMIT -n $(($SLICESIZE*$ARG1)) | awk '{printf "%-99s\n", $0}' > $FILE
+echo "$HOSTNAME $FILE"
+exit 0
\ No newline at end of file
Property changes on: SwiftApps/Swift-MapRed/mapred_chirp_combiner/old_exec_scripts/teragen_wrap_nc.sh
___________________________________________________________________
Added: svn:executable
+ *
Modified: SwiftApps/Swift-MapRed/mapred_chirp_combiner/parcombiner_c_exec.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/parcombiner_c_exec.sh 2013-09-20 02:43:24 UTC (rev 7091)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/parcombiner_c_exec.sh 2013-09-20 21:38:32 UTC (rev 7092)
@@ -21,6 +21,7 @@
EOF
chmod 777 recombiner.sh
+cp /homes/yadunand/bin/exec_client .
combine_fetch_process()
{
@@ -33,12 +34,14 @@
CHIRPDIR=$(dirname ${FILES[0]})
echo -e "#!/bin/bash \nID=$ID \nFILES=(${FILES[*]}) \n" > $SITE.$ID.sh
- cat recombiner.sh >> $SITE.$ID.sh
- cp /homes/yadunand/bin/exec_client ./
- ./exec_client $SITE $SITE.$ID.sh &> $SITE.$ID.txt
- echo "Content of $SITE.$ID.txt is $(cat $SITE.$ID.txt)"
+ cat recombiner.sh >> $SITE.$ID.sh
+ #echo "./exec_client $SITE $SITE.$ID.sh"
+ #cat $SITE.$ID.sh
+ ./exec_client $SITE $SITE.$ID.sh | tee $SITE.$ID.txt
+ #echo "Content of $SITE.$ID.txt is $(cat $SITE.$ID.txt)"
}
+echo "parcombiner_c_exec running at $PWD"
STAGE2="";
SITES=($(awk '{ print $1 }' RESULT_FILE | sort -u))
for SITE in ${SITES[*]}
Deleted: SwiftApps/Swift-MapRed/mapred_chirp_combiner/parcombiner_nc.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/parcombiner_nc.sh 2013-09-20 02:43:24 UTC (rev 7091)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/parcombiner_nc.sh 2013-09-20 21:38:32 UTC (rev 7092)
@@ -1,61 +0,0 @@
-#!/bin/bash
-
-FILES=$*
-SUM=0
-COUNT=0
-REMOTE_PORT=29900
-cat $* > RESULT_FILE
-
-# Find the unique sites in the RESULT_FILE to run a combiner there.
-#SITES=($(sort -u -t' ' -k1,1 RESULT_FILE))
-cat <<'EOF' > recombiner.sh
-SUM=0;COUNT=0;
-for file in ${FILES[*]}; do
- [ ! -f $file ] && echo "$file is missing" >> $HOSTNAME.parrot_run.log
- RES=($(awk '{ sum += $1 } END { print sum,NR }' $file))
- SUM=$(($SUM+${RES[0]}))
- COUNT=$(($COUNT+${RES[1]}))
- rm $file &
-done;
-echo -e "$SUM $COUNT"
-EOF
-chmod 777 recombiner.sh
-
-
-combine_fetch_process()
-{
- SITE=$1; shift
- ID=$1; shift
- REMOTE_PORT=$1;
- PORT=$(($REMOTE_PORT+$ID))
-
- FILES=($(grep $SITE RESULT_FILE | awk '{ print $2 }'))
- CHIRPDIR=$(dirname ${FILES[0]})
- #chirp_put ./recombiner.sh $SITE recombiner.sh
- #ssh $SITE "cd $CHIRPDIR; ./recombiner.sh $ID ${FILES[*]}"
- #chirp_get $SITE $SITE.$ID.imd $SITE.$ID.txt
-
- echo -e "$(hostname -f) $PORT\n#!/bin/bash \n ID=$ID \n FILES=(${FILES[*]}) \n" > $SITE.$ID.sh
- cat recombiner.sh >> $SITE.$ID.sh
- nc $SITE $REMOTE_PORT < $SITE.$ID.sh
- nc -l $PORT > $SITE.$ID.txt
- #rm $SITE.$ID.sh
-}
-
-STAGE2="";
-SITES=($(awk '{ print $1 }' RESULT_FILE | sort -u))
-for SITE in ${SITES[*]}
-do
- ID=$RANDOM
- combine_fetch_process $SITE $ID $REMOTE_PORT &
- STAGE2="$STAGE2 $SITE.$ID.txt"
-done;
-wait
-
-cat ${STAGE2[*]} > TEMP
-SUM=($(awk '{ sum += $1 } END { print sum }' TEMP))
-COUNT=($(awk '{ sum += $2 } END { print sum }' TEMP))
-echo "Final results (on $HOSTNAME):"
-echo "$SUM $COUNT"
-
-exit 0;
\ No newline at end of file
Modified: SwiftApps/Swift-MapRed/mapred_chirp_combiner/run.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/run.sh 2013-09-20 02:43:24 UTC (rev 7091)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/run.sh 2013-09-20 21:38:32 UTC (rev 7092)
@@ -80,7 +80,7 @@
echo "================================================================"
}
-run_swift_chirp_combiner 3 10;
+run_swift_chirp_combiner 10 10;
exit 0
####################SWIFT + CHIRP VERSION############################
Modified: SwiftApps/Swift-MapRed/mapred_chirp_combiner/server.c
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/server.c 2013-09-20 02:43:24 UTC (rev 7091)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/server.c 2013-09-20 21:38:32 UTC (rev 7092)
@@ -45,28 +45,27 @@
char exec_string[500];
int ret;
- sprintf(filename,"exec_script_%d_%d", getpid(), rand());
+ sprintf(filename,"/sandbox/yadunand/exec_script_%d", getpid());
//printf("Debug: filename: %s \n", filename);
- if ( (f_out = fopen(filename,"w")) == NULL ){
+ if ( (f_out = fopen(filename,"w+")) == NULL ){
printf("Opening[%s] failed: %s\n", filename, strerror(errno));
exit(errno);
};
fwrite(query, query_size, 1, f_out);
fclose(f_out);
- sprintf(exec_string, "chmod +x %s; ./%s",
+ sprintf(exec_string, "chmod +x %s; %s",
filename, filename);
- //printf("Command string : [%s]\n", exec_string);
+ printf("Command string : [%s]\n", exec_string);
if ( (p_exec = popen(exec_string, "r")) == NULL ){
printf("exec_string failed to popen: %s", strerror(errno));
return errno;
}
- ret = fread(reply, MAXDATASIZE, 1, p_exec);
-
- //printf("Reply is: \n%s Len = %d\n", reply, (int)strlen(reply));
+ ret = fread(reply, MAXDATASIZE, 1, p_exec);
+ printf("Reply is: \n%s Len = %d\n", reply, (int)strlen(reply));
return (strlen(reply));
}
@@ -162,12 +161,12 @@
}
buf[numbytes] = '\0';
- //printf("Server: Received string from client: \n'%s'\n",buf);
+ printf("Server: Received string from client: \n'%s'\n",buf);
// TODO
reply = (char *) malloc (2048);
reply_size = exec_string(buf, numbytes, reply);
- //printf("Returning string of size %d \n%s\n", reply_size, reply);
+ printf("Returning string of size %d \n%s\n", reply_size, reply);
if (send(new_fd, reply, reply_size, 0) == -1)
perror("send");
Deleted: SwiftApps/Swift-MapRed/mapred_chirp_combiner/server.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/server.sh 2013-09-20 02:43:24 UTC (rev 7091)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/server.sh 2013-09-20 21:38:32 UTC (rev 7092)
@@ -1,10 +0,0 @@
-#!/bin/bash
-
-PORT=20033
-TMP="tmp.$RANDOM.$HOSTNAME"
-nc -l $PORT > $TMP
-chmod +x $TMP
-RETURN_ADDRESS=$(head -n 1 $TMP)
-sed -i -e "1d" $TMP
-./$TMP &> $TMP.reply
-nc $RETURN_ADDRESS < $TMP.reply
Modified: SwiftApps/Swift-MapRed/mapred_chirp_combiner/teragen_wrap_c_exec.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/teragen_wrap_c_exec.sh 2013-09-20 02:43:24 UTC (rev 7091)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/teragen_wrap_c_exec.sh 2013-09-20 21:38:32 UTC (rev 7092)
@@ -34,7 +34,7 @@
echo "unix:$USER rwlds" > $FOLDER/acl.conf
echo "hostname:* rwl" >> $FOLDER/acl.conf
timeout $TIMEOUT chirp_server -A $FOLDER/acl.conf -r $FOLDER &
- timeout $TIMEOUT ~/bin/exec_server &
+ timeout $TIMEOUT ~/bin/exec_server &> /homes/yadunand/Swift-MapRed/mapred_chirp_combiner/exec_server_log-$HOSTNAME-$RANDOM.log &
fi
FILE=$FOLDER/$FILE;
else
Deleted: SwiftApps/Swift-MapRed/mapred_chirp_combiner/teragen_wrap_nc.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/teragen_wrap_nc.sh 2013-09-20 02:43:24 UTC (rev 7091)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/teragen_wrap_nc.sh 2013-09-20 21:38:32 UTC (rev 7092)
@@ -1,64 +0,0 @@
-#!/bin/bash
-
-
-# By default with ARG1:100 and SLICESIZE=10000, this script will generate
-# 10^6 records.
-ARG1=1
-[ ! -z $1 ] && ARG1=$1
-
-FILE="input_$RANDOM.txt"
-LOWERLIMIT=0
-UPPERLIMIT=1000000000 # 10^9
-SLICESIZE=10000 # 10^4 records padded to 100B would result in 1MB file
-#SLICESIZE=1000 # 10^3 If padded to 100B would result
-
-
-FOLDER="/sandbox/$USER"
-TIMEOUT=600
-NC_LISTEN_PORT=29900
-
-# clean_folder SLEEP_TIME FOLDER_NAME
-clean_folder()
-{
- sleep $(($1+1));
- rm $2/* &> /dev/null
-}
-
-exec_server()
-{
- cd /tmp
- PORT=$1
- TMP="/homes/yadunand/tmp.$RANDOM.$HOSTNAME"
- nc -l $PORT > $TMP
- chmod +x $TMP
- RETURN_ADDRESS=$(head -n 1 $TMP)
- sed -i -e "1d" $TMP
- ls >> $TMP.reply
- pwd >> $TMP.reply
- echo "----" >> $TMP.reply
- /bin/bash $TMP &> $TMP.reply
- echo "----" >> $TMP.reply
- echo "nc $RETURN_ADDRESS < $TMP.reply" >> $TMP.reply
- nc $RETURN_ADDRESS < $TMP.reply
-}
-
-JOBS_LEVEL=`echo $PWD | grep -o ".*jobs"`
-if [ "$?" == "0" ]
-then
- mkdir $JOBS_LEVEL/CHIRPING
- if [ "$?" == "0" ]
- then
- killall -u $USER chirp_server;
- echo "unix:$USER rwlds" > $FOLDER/acl.conf
- echo "hostname:* rwl" >> $FOLDER/acl.conf
- timeout $TIMEOUT chirp_server -A $FOLDER/acl.conf -r $FOLDER &
- exec_server $NC_LISTEN_PORT &
- fi
- FILE=$FOLDER/$FILE;
-else
- FILE=$PWD/$FILE
-fi
-
-shuf -i $LOWERLIMIT-$UPPERLIMIT -n $(($SLICESIZE*$ARG1)) | awk '{printf "%-99s\n", $0}' > $FILE
-echo "$HOSTNAME $FILE"
-exit 0
\ No newline at end of file
Deleted: SwiftApps/Swift-MapRed/mapred_chirp_combiner/test
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp_combiner/test 2013-09-20 02:43:24 UTC (rev 7091)
+++ SwiftApps/Swift-MapRed/mapred_chirp_combiner/test 2013-09-20 21:38:32 UTC (rev 7092)
@@ -1,5 +0,0 @@
-stomp /sandbox/yadunand/input_32754.txt
-thwomp /sandbox/yadunand/input_15959.txt
-stomp /sandbox/yadunand/input_21602.txt
-vanquish /sandbox/yadunand/input_18357.txt
-thrash /sandbox/yadunand/input_27031.txt
More information about the Swift-commit
mailing list