[Swift-commit] r3760 - provenancedb/apps/oops

[noreply at svn.ci.uchicago.edu]

Author: lgadelha
Date: 2010-12-10 12:12:34 -0600 (Fri, 10 Dec 2010)
New Revision: 3760

Added:
   provenancedb/apps/oops/raptor_extractor.sh
Modified:
   provenancedb/apps/oops/oops_extractor.sh
Log:
RaptorLoops annotation extractor for provenancedb.


Modified: provenancedb/apps/oops/oops_extractor.sh
===================================================================
--- provenancedb/apps/oops/oops_extractor.sh	2010-12-09 22:28:16 UTC (rev 3759)
+++ provenancedb/apps/oops/oops_extractor.sh	2010-12-10 18:12:34 UTC (rev 3760)
@@ -2,8 +2,6 @@
 
 # Annotation extractor for the OOPS application
 # 
-# The directory $PROTESTS/swift-logs contains symbolic links to 
-# OOPS' Swift logs. 
 
 PROVDB_HOME=~/provenancedb
 PROTESTS_HOME=/gpfs/pads/oops/aashish/CASP

Added: provenancedb/apps/oops/raptor_extractor.sh
===================================================================
--- provenancedb/apps/oops/raptor_extractor.sh	                        (rev 0)
+++ provenancedb/apps/oops/raptor_extractor.sh	2010-12-10 18:12:34 UTC (rev 3760)
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Annotation extractor for the OOPS application
+# 
+# The directory $PROTESTS/swift-logs contains symbolic links to 
+# OOPS' Swift logs. 
+PROVDB_HOME=~/provenancedb
+PROTESTS_HOME=/gpfs/pads/oops/aashish/CASP
+IMPORT_HOME=~/protests
+
+source $PROVDB_HOME/etc/provenance.config
+
+# provdb_imported records runs already imported to the provenance database
+cd $IMPORT_HOME
+if [ ! -a provdb_imported_raptor ]; then
+    touch provdb_imported_raptor
+fi
+
+cd $PROTESTS_HOME
+for k in $(ls -1);
+do
+    cd $PROTESTS_HOME/$k
+    for i in $(ls | grep run.raptorloops);
+    do 
+	cd $IMPORT_HOME
+	if ! grep --silent $i provdb_imported_raptor; then
+	    if grep --silent "Swift finished with no errors" $PROTESTS_HOME/$k/$i/RaptorLoops-*.log; then
+	        # swift-prov-import-all-logs also controls what already has been
+	        # imported, so it does not repeat work
+		echo "export LOGREPO=$PROTESTS_HOME/$k/$i" > $PROVDB_HOME/etc/provenance.config
+		echo "export SQLCMD=\"psql -U provdb -h db.ci.uchicago.edu provdb\"" >> $PROVDB_HOME/etc/provenance.config
+		$PROVDB_HOME/swift-prov-import-all-logs
+		cd $IMPORT_HOME
+		echo $i >> provdb_imported_raptor
+		cd swift-logs
+	        # annotate workflows with their oops runid
+		OOPS_RUN_ID=$(echo $i | awk -F . '{print $3}')
+		cd $PROTESTS_HOME/$k/$i
+		LOG_FILENAME=$(ls | grep RaptorLoops- | grep "\."log$)
+ 		WORKFLOW_ID=$(echo "select id from workflow where log_filename like '%$LOG_FILENAME%'" | $SQLCMD -t | awk '{print $1}')
+		cd $IMPORT_HOME/swift-logs
+		echo "insert into annot_wf_txt values ('$WORKFLOW_ID','oops_run_id','$OOPS_RUN_ID');" | $SQLCMD
+		
+		echo "select file.id,file.filename from process, ds_usage, ds_containment, file where  process.id=ds_usage.process_id and ds_usage.dataset_id=out_id and file.id=ds_containment.in_id and filename like '%.params' and process.name='PrepLoop' and process.workflow_id='$WORKFLOW_ID';" > query.sql;
+		
+		$SQLCMD -t -A -F " " -f query.sql -o result.txt
+		
+                DATASET_ID=$(awk '{print $1}' result.txt)
+		FILENAME=$(awk '{print $2}' result.txt | sed 's/file:\/\/localhost\///g')
+		
+		cd $PROTESTS_HOME/$k/run.raptorloops.$OOPS_RUN_ID
+		
+		while read line; do
+		    NAME=$(echo $line | awk 'BEGIN { FS = "=" }; {print $1}')
+		    if [ "$NAME" = "SAMPLE RANGE" ]; then
+			VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $1}')
+			VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $2}')
+			echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME BEGIN', $VALUE1);" | $SQLCMD
+			echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME END', $VALUE2);" | $SQLCMD
+		    fi 
+		    if [ "$NAME" = "RESTRAIN DISTANCE" ]; then
+			VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $1}')
+			VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $2}')
+			echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 1', $VALUE1);" | $SQLCMD
+			echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 2', $VALUE2);" | $SQLCMD
+		    fi 
+		    if [ "$NAME" = "MAXIMUM NUMBER OF STEPS" ]; then
+			VALUE=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}')
+			echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME', $VALUE);" | $SQLCMD
+		    fi 
+		done < $FILENAME
+	    fi
+	fi 
+    done
+done


Property changes on: provenancedb/apps/oops/raptor_extractor.sh
___________________________________________________________________
Name: svn:executable
   + *