[Swift-commit] r2778 - provenancedb

[noreply at svn.ci.uchicago.edu]

Author: benc
Date: 2009-03-30 06:54:35 -0500 (Mon, 30 Mar 2009)
New Revision: 2778

Modified:
   provenancedb/prepare-for-import
   provenancedb/prov-init.sql
   provenancedb/prov-to-sql.sh
Log:
Put execute2 records into the DB, tied to their containing execute records.

Modified: provenancedb/prepare-for-import
===================================================================
--- provenancedb/prepare-for-import	2009-03-30 05:27:28 UTC (rev 2777)
+++ provenancedb/prepare-for-import	2009-03-30 11:54:35 UTC (rev 2778)
@@ -4,7 +4,7 @@
 # workflow has finished and will do everything necessary to import all
 # of the provenance information into the SQL provenance database.
 
-swift-plot-log $1 execute.global.event execute2.event workflow.event
+swift-plot-log $1 execute.global.event execute2.event workflow.event execute2.global.event
 
 prepare-provenance-chart $1
 

Modified: provenancedb/prov-init.sql
===================================================================
--- provenancedb/prov-init.sql	2009-03-30 05:27:28 UTC (rev 2777)
+++ provenancedb/prov-init.sql	2009-03-30 11:54:35 UTC (rev 2778)
@@ -3,6 +3,7 @@
 -- implementation (in both sqlite3 and postgres)
 
 DROP TABLE executes;
+DROP TABLE execute2s;
 DROP TABLE dataset_usage;
 DROP TABLE invocation_procedure_names;
 DROP TABLE dataset_containment;
@@ -32,7 +33,19 @@
      scratch char(128)
     );
 
+-- this gives information about each execute2, which is an attempt to
+-- perform an execution. the execute2 id is tied to per-execution-attempt
+-- information such as wrapper logs
 
+CREATE TABLE execute2s
+    (id char(128) PRIMARY KEY,
+     execute_id, -- secondary key to executes table
+     starttime numeric,
+     duration numeric,
+     finalstate char(128),
+     site char(128)
+    );
+
 -- dataset_usage records usage relationships between executes and datasets;
 -- in SwiftScript terms, the input and output parameters for each
 -- application procedure invocation; in OPM terms, the artificts which are

Modified: provenancedb/prov-to-sql.sh
===================================================================
--- provenancedb/prov-to-sql.sh	2009-03-30 05:27:28 UTC (rev 2777)
+++ provenancedb/prov-to-sql.sh	2009-03-30 11:54:35 UTC (rev 2778)
@@ -1,13 +1,28 @@
 #!/bin/bash
 
-echo Generating SQL
 
+export RUNID=$(basename $1 .log)
+
+export WFID="tag:benc at ci.uchicago.edu,2008:swiftlogs:execute:${RUNID}:"
+
+echo Generating SQL for $RUNID
+
 rm -f tmp-import.sql
 
 while read time duration thread endstate app scratch; do
   echo "INSERT INTO executes (id, starttime, duration, finalstate, app, scratch) VALUES ('$thread', $time, $duration, '$endstate', '$app', '$scratch');" >> tmp-import.sql
 done < execute.global.event
 
+while read starttime duration globalid id endstate thread site scratch; do
+
+  # cut off the last component of the thread, so that we end up at the
+  # parent thread id which should correspond with the execute-level ID
+  execute_id="$WFID$(echo $thread | sed 's/-[^-]*$//')"
+
+  echo "INSERT INTO execute2s (id, execute_id, starttime, duration, finalstate, site) VALUES ('$globalid', '$execute_id', '$starttime', '$duration', '$endstate', '$site');" >> tmp-import.sql
+done < execute2.global.event
+
+
 while read thread direction dataset variable rest; do 
   if [ "$direction" == "input" ] ; then
     dir=I