[Swift-commit] r3759 - in provenancedb: . apps/oops
noreply at svn.ci.uchicago.edu
noreply at svn.ci.uchicago.edu
Thu Dec 9 16:28:16 CST 2010
Author: lgadelha
Date: 2010-12-09 16:28:16 -0600 (Thu, 09 Dec 2010)
New Revision: 3759
Modified:
provenancedb/apps/oops/oops_extractor.sh
provenancedb/swift-prov-import-all-logs
Log:
Minor updates to provenancedb.
Modified: provenancedb/apps/oops/oops_extractor.sh
===================================================================
--- provenancedb/apps/oops/oops_extractor.sh 2010-12-09 05:32:32 UTC (rev 3758)
+++ provenancedb/apps/oops/oops_extractor.sh 2010-12-09 22:28:16 UTC (rev 3759)
@@ -18,18 +18,15 @@
fi
cd $PROTESTS_HOME
-for k in `ls -1`;
+for k in $(ls -1);
do
cd $PROTESTS_HOME/$k
- for i in `ls | grep run.loops`;
+ for i in $(ls | grep run.loops);
do
cd $IMPORT_HOME
if ! grep --silent $i provdb_imported; then
if grep --silent "Swift finished with no errors" $PROTESTS_HOME/$k/$i/psim.loops-*.log; then
cd swift-logs
- #for j in `ls $PROTESTS_HOME/$k/$i | grep psim.loops-`; do
- # ln -s $PROTESTS_HOME/$k/$i/$j
- #done
cd import
# swift-prov-import-all-logs also controls what already has been
# imported, so it does not repeat work
@@ -39,45 +36,41 @@
cd $IMPORT_HOME
echo $i >> provdb_imported
cd swift-logs
- # annotate workflows with their oops runid
- OOPS_RUN_ID=`echo $i | awk -F . '{print $3}'`
+
+ # annotate workflows with their oops runid
+ OOPS_RUN_ID=$(echo $i | awk -F . '{print $3}')
cd $PROTESTS_HOME/$k/$i
- LOG_FILENAME=`ls | grep psim.loops- | grep "\."log$`
- WORKFLOW_ID=`echo "select id from workflow where log_filename like '%$LOG_FILENAME%'" | $SQLCMD -t | awk '{print $1}'`
+ LOG_FILENAME=$(ls | grep psim.loops- | grep "\."log$)
+ WORKFLOW_ID=$(echo "select id from workflow where log_filename like '%$LOG_FILENAME%'" | $SQLCMD -t | awk '{print $1}')
cd $IMPORT_HOME/swift-logs
echo "insert into annot_wf_txt (id, name, value) values ('$WORKFLOW_ID','oops_run_id','$OOPS_RUN_ID');" | $SQLCMD
- echo "select id,filename from file where filename like '%params%' and id in (select in_id from ds_containment where out_id in (select ds_usage.dataset_id from ds_usage,process,execute where ds_usage.process_id=process.id and process.id=execute.id and execute.procedure_name='loopPrepare' and ds_usage.direction='I' and process.workflow_id like '%$WORKFLOW_ID%'));" > query.sql
+ #
+ echo "select file.id,file.filename from process, ds_usage, ds_containment, file where process.id=ds_usage.process_id and ds_usage.dataset_id=out_id and file.id=ds_containment.in_id and filename like '%.params' and process.name='PrepLoop' and process.workflow_id='$WORKFLOW_ID';" > query.sql;
- #query for the previous database schema
- #echo "select dataset_id,filename from dataset_filenames where filename like '%params%' and dataset_id in (select inner_dataset_id from dataset_containment where outer_dataset_id in (select dataset_usage.dataset_id from invocation_procedure_names,dataset_usage,processes_in_workflows where invocation_procedure_names.execute_id=dataset_usage.process_id and dataset_usage.process_id=processes_in_workflows.process_id and invocation_procedure_names.procedure_name='loopPrepare' and dataset_usage.direction='I' and processes_in_workflows.workflow_id like '%$WORKFLOW_ID%'));" > query.sql
-
$SQLCMD -t -A -F " " -f query.sql -o result.txt
- #DATASET_ID=`awk '{print $1}' result.txt`
- DATASET_ID=`awk '{if (NR==1) print $1}' result.txt`
+ DATASET_ID=$(awk '{print $1}' result.txt)
+ FILENAME=$(awk '{print $2}' result.txt | sed 's/file:\/\/localhost\///g')
- #FILENAME=`awk '{print $2}' result.txt | sed 's/file:\/\/localhost\///g'`
- FILENAME=`awk '{if (NR==1) print $2}' result.txt | sed 's/file:\/\/localhost\///g'`
-
cd $PROTESTS_HOME/$k/run.loops.$OOPS_RUN_ID
while read line; do
- NAME=`echo $line | awk 'BEGIN { FS = "=" }; {print $1}'`
+ NAME=$(echo $line | awk 'BEGIN { FS = "=" }; {print $1}')
if [ "$NAME" = "SAMPLE RANGE" ]; then
- VALUE1=`echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $1}'`
- VALUE2=`echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $2}'`
+ VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $1}')
+ VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $2}')
echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME BEGIN', $VALUE1);" | $SQLCMD
echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME END', $VALUE2);" | $SQLCMD
fi
if [ "$NAME" = "RESTRAIN DISTANCE" ]; then
- VALUE1=`echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $1}'`
- VALUE2=`echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $2}'`
+ VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $1}')
+ VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $2}')
echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 1', $VALUE1);" | $SQLCMD
echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 2', $VALUE2);" | $SQLCMD
fi
if [ "$NAME" = "MAXIMUM NUMBER OF STEPS" ]; then
- VALUE=`echo $line | awk 'BEGIN { FS = "=" }; {print $2}'`
+ VALUE=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}')
echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME', $VALUE);" | $SQLCMD
fi
done < $FILENAME
Modified: provenancedb/swift-prov-import-all-logs
===================================================================
--- provenancedb/swift-prov-import-all-logs 2010-12-09 05:32:32 UTC (rev 3758)
+++ provenancedb/swift-prov-import-all-logs 2010-12-09 22:28:16 UTC (rev 3759)
@@ -41,7 +41,7 @@
EXISTING=$($SQLCMD --tuples-only -c "select count(*) from workflow where log_filename='$filename';")
if [ "$EXISTING" -eq "0" ]; then
- PROV_ENABLED=$(grep provenance $filename | wc -l)
+ PROV_ENABLED=$(grep provenanceid $filename | wc -l)
if [ $PROV_ENABLED -gt 0 ]; then
echo IMPORTING
More information about the Swift-commit
mailing list