[Swift-commit] r3335 - in provenancedb: . apps apps/oops

[noreply at svn.ci.uchicago.edu]

Author: lgadelha
Date: 2010-05-27 16:33:34 -0500 (Thu, 27 May 2010)
New Revision: 3335

Added:
   provenancedb/apps/
   provenancedb/apps/oops/
   provenancedb/apps/oops/oops_extractor.sh
Modified:
   provenancedb/swift-prov-import-all-logs
Log:
Started scientific parameter extractor for the OOPS application


Added: provenancedb/apps/oops/oops_extractor.sh
===================================================================
--- provenancedb/apps/oops/oops_extractor.sh	                        (rev 0)
+++ provenancedb/apps/oops/oops_extractor.sh	2010-05-27 21:33:34 UTC (rev 3335)
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# Annotation extractor for the OOPS application
+# Author: Luiz Gadelha
+# Date: 2010-05-25
+# 
+# The directory $PROTESTS/swift-logs contains symbolic links to 
+# OOPS' Swift logs. 
+
+PROVDB_HOME=~/provenancedb
+PROTESTS_HOME=~/protests
+workflow_id=$1
+
+source $PROVDB_HOME/etc/provenance.config
+
+
+# provdb_imported records runs already imported to the provenance database
+cd $PROTESTS_HOME
+if [ ! -a provdb_imported ]; then
+    touch provdb_imported
+fi
+
+
+for i in `ls | grep run.loops`;
+do 
+    if ! grep $i provdb_imported; then
+	if grep "Swift finished with no errors" $i/psim.loops-*.log; then
+	    cd swift-logs
+	    for j in `ls ../$i | grep psim.loops-`; do
+		ln -s ../$i/$j
+	    done
+	    cd import
+	    # swift-prov-import-all-logs also controls what already has been
+	    # imported, so it does not repeat work
+	    $PROVDB_HOME/swift-prov-import-all-logs
+	    cd $PROTESTS_HOME
+	    echo $i >> provdb_imported
+
+	    # annotate workflows with their oops runid
+	    oops_run_id=`echo $i | awk -F . '{print $3}'`
+	    log_filename=`ls $i | grep psim.loops- | grep "\."log$`
+	    workflow_id=`echo "select workflow_id from known_workflows where workflow_log_filename like '%$log_filename%'" | $SQLCMD -t | awk '{print $1}'`
+	    echo "insert into annotations values ('$workflow_id','oops_run_id','$oops_run_id');" | $SQLCMD
+
+	    # annotate dataset with scientific parameters passed to doLoopRound
+	    #echo "\pset border 0;" > query.sql
+            #echo "select dataset_filenames.dataset_id,dataset_filenames.filename from dataset_usage,invocation_procedure_names,dataset_containment,dataset_filenames where dataset_usage.process_id=invocation_procedure_names.execute_id and dataset_containment.inner_dataset_id=dataset_filenames.dataset_id and procedure_name='doLoopRound' and param_name='modelData' and dataset_containment.outer_dataset_id=dataset_usage.dataset_id and dataset_filenames.filename like '%.params%';" >> query.sql
+            #$SQLCMD -t -A -F " " -f query.sql -o result.txt
+            #dataset_id=`awk '{print $1}' result.txt`
+            #filename=`awk '{print $2}' result.txt | sed 's/file:\/\/localhost\///g'`
+            #TODO extract name-value pairs
+	fi 
+    fi 
+done
+
+
+


Property changes on: provenancedb/apps/oops/oops_extractor.sh
___________________________________________________________________
Name: svn:executable
   + *

Modified: provenancedb/swift-prov-import-all-logs
===================================================================
--- provenancedb/swift-prov-import-all-logs	2010-05-24 21:28:38 UTC (rev 3334)
+++ provenancedb/swift-prov-import-all-logs	2010-05-27 21:33:34 UTC (rev 3335)
@@ -35,41 +35,40 @@
  if [ $version -ge 1538 ]; then
    echo -n "Log: $filename ... "
 
-# TODO: this only works for psql, not for other DBs, so for now force it
-# to be non-existing
-#EXISTING=$(psql -p 5435 -d $PROVDB -U benc --tuples-only -c "select count(*) from known_workflows where workflow_log_filename='$filename';")
-EXISTING=0
+# TODO: make it work in general (sqlite, mysql, ...)
+# Works only with PostgreSQL
+EXISTING=$($SQLCMD --tuples-only -c "select count(*) from known_workflows where workflow_log_filename='$filename';")
 
 if [ "$EXISTING" -eq "0" ];  then
-echo IMPORTING
-
-   if grep --silent "DEBUG Loader Swift finished with no errors" $filename; then
-     wfstatus="SUCCESS"
-   else
-     wfstatus="FAIL"
-   fi
-
-   echo version $version in log file $filename
-     echo ============= will import =============
-     prepare-for-import $filename
-     if [ "$?" != "0" ]; then
-       echo prepare-for-import failed
-       exit 2
-     fi
-     import-run-to-sql $filename
-     if [ "$?" != "0" ]; then
-       echo import-run-to-sql failed
-       exit 3
-     fi
-
+    echo IMPORTING
+    
+    if grep --silent "DEBUG Loader Swift finished with no errors" $filename; then
+	wfstatus="SUCCESS"
+    else
+	wfstatus="FAIL"
+    fi
+    
+    echo version $version in log file $filename
+    echo ============= will import =============
+    prepare-for-import $filename
+    if [ "$?" != "0" ]; then
+	echo prepare-for-import failed
+	exit 2
+    fi
+    import-run-to-sql $filename
+    if [ "$?" != "0" ]; then
+	echo import-run-to-sql failed
+	exit 3
+    fi
+    
 #     import-run-to-xml $filename
-
-     export RUNID=$(basename $filename .log)
-     export WF="tag:benc at ci.uchicago.edu,2008:swiftlogs:execute:${RUNID}:run"
-
-     echo "INSERT INTO known_workflows (workflow_id, workflow_log_filename, version, importstatus) VALUES ('$WF','$filename','$version','$wfstatus');" | $SQLCMD
+    
+    export RUNID=$(basename $filename .log)
+    export WF="tag:benc at ci.uchicago.edu,2008:swiftlogs:execute:${RUNID}:run"
+    
+    echo "INSERT INTO known_workflows (workflow_id, workflow_log_filename, version, importstatus) VALUES ('$WF','$filename','$version','$wfstatus');" | $SQLCMD
 else
-echo SKIP: Already known in workflow
+    echo SKIP: Already known in workflow
 fi
  fi
 done < /tmp/everylog-vs-versions.data