[Swift-commit] r6108 - provenancedb
lgadelha at ci.uchicago.edu
lgadelha at ci.uchicago.edu
Wed Dec 19 07:58:29 CST 2012
Author: lgadelha
Date: 2012-12-19 07:57:43 -0600 (Wed, 19 Dec 2012)
New Revision: 6108
Modified:
provenancedb/README.asciidoc
provenancedb/info-to-runtime
provenancedb/prov-init.sql
provenancedb/prov-to-sql.sh
Log:
Update to provenance extraction scripts to match new format of runtime information in wrapper logs.
Modified: provenancedb/README.asciidoc
===================================================================
--- provenancedb/README.asciidoc 2012-12-18 00:15:14 UTC (rev 6107)
+++ provenancedb/README.asciidoc 2012-12-19 13:57:43 UTC (rev 6108)
@@ -188,10 +188,15 @@
=== Example: MODIS
-Run MODIS.
+Run MODIS:
--------------------------------------
swift modis.swift
+--------------------------------------
+
+After running MODIS, one can import provenenace information into the database using:
+
+--------------------------------------
swift-prov-import-all-logs
--------------------------------------
@@ -213,8 +218,8 @@
modis.swift | 5483 | 3339 | SUCCESS | 2012-10-26 11:44:59.909-02 | 85.050
--------------------------------------
+List the
-
--------------------------------------
select * from ancestors('dataset:20121026-1146-jng6bir4:720000001604');
Modified: provenancedb/info-to-runtime
===================================================================
--- provenancedb/info-to-runtime 2012-12-18 00:15:14 UTC (rev 6107)
+++ provenancedb/info-to-runtime 2012-12-19 13:57:43 UTC (rev 6108)
@@ -10,7 +10,7 @@
if [ "X$record" != "X" ] && [ -f $record ] ; then
- grep '^RUNTIME_INFO=' $record | sed "s/^RUNTIME_INFO=\(.*\)$/$globalid \1/"
+ grep '^APP_RESOURCES=' $record | sed "s/^APP_RESOURCES=\(.*\)$/$globalid \1/"
else
echo no wrapper log for $id >&2
Modified: provenancedb/prov-init.sql
===================================================================
--- provenancedb/prov-init.sql 2012-12-18 00:15:14 UTC (rev 6107)
+++ provenancedb/prov-init.sql 2012-12-19 13:57:43 UTC (rev 6108)
@@ -117,8 +117,31 @@
start_time numeric,
duration numeric,
final_state varchar(32),
- site varchar(256)
+ site varchar(256),
+ real_secs numeric,
+ kernel_secs numeric,
+ user_secs numeric,
+ percent_cpu numeric,
+ max_rss numeric,
+ avg_rss numeric,
+ avg_tot_vm numeric,
+ avg_priv_data numeric,
+ avg_priv_stack numeric,
+ avg_shared_text numeric,
+ page_size numeric,
+ major_pgfaults numeric,
+ minor_pgfaults numeric,
+ swaps numeric,
+ invol_context_switches numeric,
+ vol_waits numeric,
+ fs_reads numeric,
+ fs_writes numeric,
+ sock_recv numeric,
+ sock_send numeric,
+ signals numeric,
+ exit_status numeric
);
+
-- maxrss numeric,
-- walltime numeric,
-- systime numeric,
@@ -291,7 +314,13 @@
);
create view provenance_graph_edge as
- select function_call_id as parent, dataset_id as child from dataset_out
- union all
- select dataset_id as parent, function_call_id as child from dataset_in;
+ select function_call_id as parent, dataset_id as child
+ from dataset_out
+ union all
+ select dataset_id as parent, function_call_id as child
+ from dataset_in;
+create view script_run_summary as
+ select id,swift_version,cog_version,final_state,
+ start_time,duration,script_filename
+ from script_run;
\ No newline at end of file
Modified: provenancedb/prov-to-sql.sh
===================================================================
--- provenancedb/prov-to-sql.sh 2012-12-18 00:15:14 UTC (rev 6107)
+++ provenancedb/prov-to-sql.sh 2012-12-19 13:57:43 UTC (rev 6108)
@@ -129,13 +129,31 @@
echo " - Wrapper log resource consumption info."
if [ -f runtime.txt ]; then
while read execute2_id runtime; do
- timestamp=$(echo $runtime | awk -F "," '{print $1}' | awk -F ":" '{print $2}')
- cpu_usage=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
- max_phys_mem=$(echo $runtime | awk -F "," '{print $3}' | awk -F ":" '{print $2}')
- max_virtual_mem=$(echo $runtime | awk -F "," '{print $4}' | awk -F ":" '{print $2}')
- io_read_bytes=$(echo $runtime | awk -F "," '{print $5}' | awk -F ":" '{print $2}')
- io_write_bytes=$(echo $runtime | awk -F "," '{print $6}' | awk -F ":" '{print $2}')
- echo "INSERT INTO rt_info (app_exec_id, timestamp, cpu_usage, max_phys_mem, max_virt_mem, io_read, io_write) VALUES ('$execute2_id', $timestamp, $cpu_usage, $max_phys_mem, $max_virtual_mem, $io_read_bytes, $io_write_bytes);" >> /tmp/$RUNID.sql
+ real_secs=$(echo $runtime | awk -F "," '{print $1}' | awk -F ":" '{print $2}')
+ kernel_secs=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ user_secs=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ percent_cpu=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ max_rss=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ avg_rss=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ avg_tot_vm=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ avg_priv_data=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ avg_priv_stack=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ avg_shared_text=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ page_size=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ major_pgfaults=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ minor_pgfaults=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ swaps=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ invol_context_switches=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ vol_waits=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ fs_reads=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ fs_writes=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ sock_recv=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ sock_send=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ signals=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+ exit_status=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
+
+ echo "UPDATE app_exec SET real_secs='$real_secs', kernel_secs='$kernel_secs', user_secs='$user_secs', percent_cpu='$percent_cpu', max_rss='$max_rss', avg_rss='$avg_rss', avg_tot_vm='$avg_tot_vm', avg_priv_data='$avg_priv_data', avg_priv_stack='$avg_priv_stack', avg_priv_stack='$avg_priv_stack', avg_shared_text='$avg_shared_text', page_size='$page_size', major_pgfaults='$major_pgfaults', minor_pgfaults='$minor_pgfaults', swaps='$swaps', invol_context_switches='$invol_context_switches', vol_waits='$vol_waits', fs_reads='$fs_reads', fs_writes='$fs_writes', sock_recv='$sock_recv', sock_send='$sock_send', signals='$signals', exit_status='$exit_status' WHERE app_exec_id='$execute2_id';"
+ #echo "INSERT INTO rt_info (app_exec_id, timestamp, cpu_usage, max_phys_mem, max_virt_mem, io_read, io_write) VALUES ('$execute2_id', $timestamp, $cpu_usage, $max_phys_mem, $max_virtual_mem, $io_read_bytes, $io_write_bytes);" >> /tmp/$RUNID.sql
# for key in $(echo maxrss walltime systime usertime cpu fsin fsout timesswapped socketrecv socketsent majorpagefaults minorpagefaults contextswitchesinv contextswitchesvol); do
# value=$(echo $runtime | awk -F "," '{print $1}' | awk -F ":" '{print $2}')
More information about the Swift-commit
mailing list