0.93 RC5 current version
2011/11/14
+
0.93 RC6 current version
2012/01/03
Read our Quick Start Guide
and start using Swift today!
From davidk at ci.uchicago.edu Tue Jan 3 16:02:39 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Tue, 3 Jan 2012 16:02:39 -0600 (CST)
Subject: [Swift-commit] r5481 - in branches/release-0.93: docs
docs/siteguide docs/siteguide/.cache docs/userguide/.cache
etc libexec tests/sites/mcs
Message-ID: <20120103220239.79C459CCB6@svn.ci.uchicago.edu>
Author: davidk
Date: 2012-01-03 16:02:39 -0600 (Tue, 03 Jan 2012)
New Revision: 5481
Modified:
branches/release-0.93/docs/build_docs.sh
branches/release-0.93/docs/siteguide/.cache/siteguide.txt
branches/release-0.93/docs/siteguide/siteguide.txt
branches/release-0.93/docs/userguide/.cache/userguide.txt
branches/release-0.93/etc/coaster-service.conf
branches/release-0.93/libexec/release.txt
branches/release-0.93/tests/sites/mcs/coaster-service.conf
Log:
Various doc updates
Modified: branches/release-0.93/docs/build_docs.sh
===================================================================
--- branches/release-0.93/docs/build_docs.sh 2012-01-03 16:42:59 UTC (rev 5480)
+++ branches/release-0.93/docs/build_docs.sh 2012-01-03 22:02:39 UTC (rev 5481)
@@ -40,7 +40,7 @@
do
pushd $directory > /dev/null 2>&1
FILES=`ls -1 *.txt 2>/dev/null`
- CONTENTFILES=`/bin/ls -l | grep ^- | awk '{print $9}' | grep -v "\."`
+ CONTENTFILES=`find . -maxdepth 1 -type f ! -iname *.pdf`
for file in $FILES
do
@@ -53,11 +53,6 @@
doflag=1
fi
done
- #diff $file .cache/$file >/dev/null 2>/dev/null
- #if [ $? -ne 0 ]
- #then
- # echo "updating cache"
- # cp $file .cache
if [ $doflag -eq 1 ]
then
echo "updating cache"
Modified: branches/release-0.93/docs/siteguide/.cache/siteguide.txt
===================================================================
--- branches/release-0.93/docs/siteguide/.cache/siteguide.txt 2012-01-03 16:42:59 UTC (rev 5480)
+++ branches/release-0.93/docs/siteguide/.cache/siteguide.txt 2012-01-03 22:02:39 UTC (rev 5481)
@@ -8,14 +8,17 @@
include::prereqs[]
-include::pads[]
+include::bag_of_workstations[]
+include::beagle[]
+
include::fusion[]
-include::beagle[]
-
include::futuregrid[]
+include::intrepid[]
+
include::mcs[]
-include::intrepid[]
+include::pads[]
+
Modified: branches/release-0.93/docs/siteguide/siteguide.txt
===================================================================
--- branches/release-0.93/docs/siteguide/siteguide.txt 2012-01-03 16:42:59 UTC (rev 5480)
+++ branches/release-0.93/docs/siteguide/siteguide.txt 2012-01-03 22:02:39 UTC (rev 5481)
@@ -8,14 +8,17 @@
include::prereqs[]
-include::pads[]
+include::bag_of_workstations[]
+include::beagle[]
+
include::fusion[]
-include::beagle[]
-
include::futuregrid[]
+include::intrepid[]
+
include::mcs[]
-include::intrepid[]
+include::pads[]
+
Modified: branches/release-0.93/docs/userguide/.cache/userguide.txt
===================================================================
--- branches/release-0.93/docs/userguide/.cache/userguide.txt 2012-01-03 16:42:59 UTC (rev 5480)
+++ branches/release-0.93/docs/userguide/.cache/userguide.txt 2012-01-03 22:02:39 UTC (rev 5481)
@@ -39,3 +39,5 @@
include::cdm[]
include::log-processing[]
+
+include::troubleshooting[]
Modified: branches/release-0.93/etc/coaster-service.conf
===================================================================
--- branches/release-0.93/etc/coaster-service.conf 2012-01-03 16:42:59 UTC (rev 5480)
+++ branches/release-0.93/etc/coaster-service.conf 2012-01-03 22:02:39 UTC (rev 5481)
@@ -1,6 +1,3 @@
-# Keep all interesting settings in one place
-# User should modify this to fit environment
-
# Location of SWIFT. If empty, PATH is searched
export SWIFT=
@@ -31,7 +28,7 @@
# This is the IP address to which the workers will connect
# If not given, start-coaster-service tries to automatically detect
-# the IP address of this system via ifconfig
+# the IP address of this system via ifconfig
# Specify this if you have multiple network interfaces
export IPADDR=
@@ -40,6 +37,3 @@
# Swift information for creating sites.xml
export WORK=$HOME/swiftwork
-export QUEUE=prod-devel
-export MAXTIME=20
-export NODE=64
Modified: branches/release-0.93/libexec/release.txt
===================================================================
--- branches/release-0.93/libexec/release.txt 2012-01-03 16:42:59 UTC (rev 5480)
+++ branches/release-0.93/libexec/release.txt 2012-01-03 22:02:39 UTC (rev 5481)
@@ -1 +1 @@
-0.93RC5
+0.93RC6
Modified: branches/release-0.93/tests/sites/mcs/coaster-service.conf
===================================================================
--- branches/release-0.93/tests/sites/mcs/coaster-service.conf 2012-01-03 16:42:59 UTC (rev 5480)
+++ branches/release-0.93/tests/sites/mcs/coaster-service.conf 2012-01-03 22:02:39 UTC (rev 5481)
@@ -28,9 +28,3 @@
# start-coaster-service tries to automatically detect IP address.
# Specify here if auto detection is not working correctly
export IPADDR=
-
-# Below are various settings to give information about how to create sites.xml
-export work=$HOME/work
-export queue=prod-devel
-export maxtime=20
-export nodes=64
From davidk at ci.uchicago.edu Tue Jan 3 16:17:23 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Tue, 3 Jan 2012 16:17:23 -0600 (CST)
Subject: [Swift-commit] r5482 - branches/release-0.93/etc
Message-ID: <20120103221723.676659CCB6@svn.ci.uchicago.edu>
Author: davidk
Date: 2012-01-03 16:17:23 -0600 (Tue, 03 Jan 2012)
New Revision: 5482
Modified:
branches/release-0.93/etc/coaster-service.conf
Log:
Update default coaster-service with jobs per node and job throttle values
Modified: branches/release-0.93/etc/coaster-service.conf
===================================================================
--- branches/release-0.93/etc/coaster-service.conf 2012-01-03 22:02:39 UTC (rev 5481)
+++ branches/release-0.93/etc/coaster-service.conf 2012-01-03 22:17:23 UTC (rev 5482)
@@ -37,3 +37,10 @@
# Swift information for creating sites.xml
export WORK=$HOME/swiftwork
+
+# Jobs per node is usually equal to the number of CPUs per node
+export JOBS_PER_NODE=2
+
+# To calculate job throttle:
+# jobs per node * nodes - 0.1 / 100
+export JOB_THROTTLE=0.059
From davidk at ci.uchicago.edu Tue Jan 3 16:19:15 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Tue, 3 Jan 2012 16:19:15 -0600 (CST)
Subject: [Swift-commit] r5483 - branches/release-0.93/docs/siteguide
Message-ID: <20120103221915.295FB9CCB6@svn.ci.uchicago.edu>
Author: davidk
Date: 2012-01-03 16:19:14 -0600 (Tue, 03 Jan 2012)
New Revision: 5483
Added:
branches/release-0.93/docs/siteguide/bag_of_workstations
Log:
Add siteguide entry for bag of workstations configuration
Added: branches/release-0.93/docs/siteguide/bag_of_workstations
===================================================================
--- branches/release-0.93/docs/siteguide/bag_of_workstations (rev 0)
+++ branches/release-0.93/docs/siteguide/bag_of_workstations 2012-01-03 22:19:14 UTC (rev 5483)
@@ -0,0 +1,109 @@
+?Bag of Workstations
+-------------------
+"Bag of workstations" refers to a collection of machines that are not connected
+together as part of a cluster or supercomputer.
+
+Prerequisites
+~~~~~~~~~~~~~
+In order to run Swift on a bag of workstations, the following prerequisites must
+be met:
+
+- The machines must be running Linux
+- The machines must have Perl available
+- A user account must be created on each machine (the username you create must be the same on each machine)
+- You must be able to SSH into the accounts without being prompted for a password. This usually involves creating an SSH key and setting up your authorized_keys. More information on how to do this can be found at http://www.openssh.org.
+
+Create a coaster-service.conf
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+To begin, copy the text below and paste it into the directory
+where your swift script is located. Name this file coaster-service.conf.
+
+-----
+include::../../etc/coaster-service.conf[]
+-----
+
+Modify coaster-service.conf
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The coaster-service.conf file contains information about your setup.
+There are a few settings you must customize.
+
+The first is the name of the machines which will be used as workers. Modify
+the line below to reflect the names of the machines you want to use.
+-----
+export WORKER_HOSTS="myhost1.mydomain myhost2.mydomain"
+-----
+
+Update the value for JOBS_PER_NODE to reflect the number of CPUs available per node.
+
+The JOB_THROTTLE setting determines the maximum number of active jobs. Use the formula
+to determine the ideal value:
+
+jobs per node * number of nodes - 0.1 / 100
+
+Example: Suppose you have 10 machines each with 8 cores
+-----
+8 jobs per node * 10 cores = 80
+80 - 0.1 = 79.9
+79.9 / 100 = 0.799
+-----
+
+In this example, you would set the job throttle to 0.799
+-----
+export JOB_THROTTLE=0.799
+-----
+
+By default, this setup assumes there are no firewall rescrictions. If there
+is a firewall rescricting SSH access, set tunneling to true with this command
+
+-----
+export SSH_TUNNELING=yes
+-----
+
+This setup also assumes that these machines are not using a shared filesystem (NFS/AFS/CIFS, etc).
+If these systems are all sharing a common filesystem, add the setting below.
+
+-----
+export SHARED_FILESYSTEM=no
+-----
+
+Starting the Coaster Service
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Change directories to the location where you would like to run a
+Swift script and start the coaster service with this
+command:
+
+-----
+start-coaster-service
+-----
+
+This will create a configuration file that Swift needs
+called sites.xml.
+
+WARNING: Any existing sites.xml files in this directory
+will be overwritten. Be sure to make a copy of any
+custom configuration files you may have.
+
+Run Swift
+~~~~~~~~~
+
+Next, run Swift. If you do not have a particular script
+in mind, you can test Swift by using a Swift script in
+the examples/ directory.
+
+Run the following command to run the script:
+-----
+swift -sites.file sites.xml -tc.file tc.data yourscript.swift
+-----
+
+Stopping the Coaster Service
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The coaster service will run indefinitely. The stop-coaster-service
+script will terminate the coaster service.
+
+-----
+$ stop-coaster-service
+-----
+
+This will kill the coaster service and kill the worker scripts on remote systems.
+
From davidk at ci.uchicago.edu Thu Jan 5 12:15:46 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Thu, 5 Jan 2012 12:15:46 -0600 (CST)
Subject: [Swift-commit] r5484 - branches/release-0.93/libexec
Message-ID: <20120105181546.D0C629CFD2@svn.ci.uchicago.edu>
Author: davidk
Date: 2012-01-05 12:15:46 -0600 (Thu, 05 Jan 2012)
New Revision: 5484
Modified:
branches/release-0.93/libexec/release.txt
Log:
Updating release
Modified: branches/release-0.93/libexec/release.txt
===================================================================
--- branches/release-0.93/libexec/release.txt 2012-01-03 22:19:14 UTC (rev 5483)
+++ branches/release-0.93/libexec/release.txt 2012-01-05 18:15:46 UTC (rev 5484)
@@ -1 +1 @@
-0.93RC6
+0.93
From davidk at ci.uchicago.edu Thu Jan 5 12:31:14 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Thu, 5 Jan 2012 12:31:14 -0600 (CST)
Subject: [Swift-commit] r5485 - in www: downloads inc main
Message-ID: <20120105183114.9FF0C9CFD2@svn.ci.uchicago.edu>
Author: davidk
Date: 2012-01-05 12:31:14 -0600 (Thu, 05 Jan 2012)
New Revision: 5485
Modified:
www/downloads/index.php
www/inc/downloads_sidebar.php
www/main/index.php
Log:
Update version
Modified: www/downloads/index.php
===================================================================
--- www/downloads/index.php 2012-01-05 18:15:46 UTC (rev 5484)
+++ www/downloads/index.php 2012-01-05 18:31:14 UTC (rev 5485)
@@ -37,15 +37,15 @@
Latest Release
-
Swift 0.93 RC6 - 2012/01/03
+
Swift 0.93 - 2012/01/05
For the majority of users, this is the version you will want to download. This package
contains the latest Stable release of Swift. Since Swift is written in Java, this
package will run on all supported platforms with Java Runtime Environment 1.5 or greater.
Precompiled binary distribution
-[swift-0.93RC6.tar.gz]
+[swift-0.93.tar.gz]
Source Code
Modified: www/inc/downloads_sidebar.php
===================================================================
--- www/inc/downloads_sidebar.php 2012-01-05 18:15:46 UTC (rev 5484)
+++ www/inc/downloads_sidebar.php 2012-01-05 18:31:14 UTC (rev 5485)
@@ -1,7 +1,7 @@
Latest Release
-
-
0.93 RC6 current version
2012/01/03
+
+
0.93 current version
2012/01/05
Modified: www/main/index.php
===================================================================
--- www/main/index.php 2012-01-05 18:15:46 UTC (rev 5484)
+++ www/main/index.php 2012-01-05 18:31:14 UTC (rev 5485)
@@ -62,7 +62,7 @@
-
0.93 RC6 current version
2012/01/03
+
0.93 current version
2012/01/05
Read our Quick Start Guide
and start using Swift today!
From davidk at ci.uchicago.edu Thu Jan 5 12:41:07 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Thu, 5 Jan 2012 12:41:07 -0600 (CST)
Subject: [Swift-commit] r5486 - tags
Message-ID: <20120105184107.BED3E9CFD2@svn.ci.uchicago.edu>
Author: davidk
Date: 2012-01-05 12:41:07 -0600 (Thu, 05 Jan 2012)
New Revision: 5486
Removed:
tags/release-0.93/
Log:
Removing old 0.93 tag
From davidk at ci.uchicago.edu Thu Jan 5 12:42:42 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Thu, 5 Jan 2012 12:42:42 -0600 (CST)
Subject: [Swift-commit] r5487 - tags
Message-ID: <20120105184242.D700B9CFD2@svn.ci.uchicago.edu>
Author: davidk
Date: 2012-01-05 12:42:42 -0600 (Thu, 05 Jan 2012)
New Revision: 5487
Added:
tags/release-0.93/
Log:
Tagging 0.93
From davidk at ci.uchicago.edu Sat Jan 7 01:48:37 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Sat, 7 Jan 2012 01:48:37 -0600 (CST)
Subject: [Swift-commit] r5489 - SwiftApps/modis
Message-ID: <20120107074837.76ED09CFAC@svn.ci.uchicago.edu>
Author: davidk
Date: 2012-01-07 01:48:36 -0600 (Sat, 07 Jan 2012)
New Revision: 5489
Added:
SwiftApps/modis/cf.local
SwiftApps/modis/cf.pads
SwiftApps/modis/demo.local
SwiftApps/modis/demo.pads
SwiftApps/modis/modis.swift
SwiftApps/modis/sites.pads.xml
SwiftApps/modis/tc.pads
Removed:
SwiftApps/modis/cf
SwiftApps/modis/demo5
SwiftApps/modis/demo5.local
SwiftApps/modis/modis5.swift
SwiftApps/modis/selectedtiles.txt
SwiftApps/modis/tc
Log:
Some updates to run modis on pads
Deleted: SwiftApps/modis/cf
===================================================================
--- SwiftApps/modis/cf 2012-01-06 21:39:41 UTC (rev 5488)
+++ SwiftApps/modis/cf 2012-01-07 07:48:36 UTC (rev 5489)
@@ -1,9 +0,0 @@
-wrapperlog.always.transfer=true
-sitedir.keep=true
-execution.retries=0
-lazy.errors=false
-status.mode=provider
-use.provider.staging=false
-provider.staging.pin.swiftfiles=false
-#throttle.transfers=16
-#throttle.file.operations=16
Added: SwiftApps/modis/cf.local
===================================================================
--- SwiftApps/modis/cf.local (rev 0)
+++ SwiftApps/modis/cf.local 2012-01-07 07:48:36 UTC (rev 5489)
@@ -0,0 +1,9 @@
+wrapperlog.always.transfer=true
+sitedir.keep=true
+execution.retries=0
+lazy.errors=false
+status.mode=provider
+use.provider.staging=false
+provider.staging.pin.swiftfiles=false
+#throttle.transfers=16
+#throttle.file.operations=16
Added: SwiftApps/modis/cf.pads
===================================================================
--- SwiftApps/modis/cf.pads (rev 0)
+++ SwiftApps/modis/cf.pads 2012-01-07 07:48:36 UTC (rev 5489)
@@ -0,0 +1,7 @@
+wrapperlog.always.transfer=true
+sitedir.keep=true
+execution.retries=0
+lazy.errors=false
+status.mode=provider
+use.provider.staging=false
+provider.staging.pin.swiftfiles=false
Added: SwiftApps/modis/demo.local
===================================================================
--- SwiftApps/modis/demo.local (rev 0)
+++ SwiftApps/modis/demo.local 2012-01-07 07:48:36 UTC (rev 5489)
@@ -0,0 +1,15 @@
+# demo4 forest 50 5 pbs
+
+landtype=${1:-urban}
+nfiles=${2:-999}
+nselect=${3:-10}
+site=${4:-local}
+
+runid=modis-$(date +%Y.%m%d.%H%M)-$landtype-$nfiles-$nselect
+
+echo runid=$runid
+
+swift -tc.file tc.local \
+ -sites.file sites.local.xml \
+ modis.swift -landtype=$landtype -nfiles=$nfiles -nselect=$nselect -runid=$runid \
+ -modisdir=$PWD/data/modis/2002 -webdir=$HOME/public_html/geo
Property changes on: SwiftApps/modis/demo.local
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/modis/demo.pads
===================================================================
--- SwiftApps/modis/demo.pads (rev 0)
+++ SwiftApps/modis/demo.pads 2012-01-07 07:48:36 UTC (rev 5489)
@@ -0,0 +1,15 @@
+# demo4 forest 50 5 pbs
+
+landtype=${1:-urban}
+nfiles=${2:-999}
+nselect=${3:-10}
+site=${4:-local}
+
+runid=modis-$(date +%Y.%m%d.%H%M)-$landtype-$nfiles-$nselect
+
+echo runid=$runid
+
+swift -tc.file tc.pads \
+ -sites.file sites.pads.xml \
+ modis.swift -config cf.pads -landtype=$landtype -nfiles=$nfiles -nselect=$nselect -runid=$runid \
+ -modisdir=$PWD/data/modis/2002 -webdir=$HOME/public_html/geo
Property changes on: SwiftApps/modis/demo.pads
___________________________________________________________________
Added: svn:executable
+ *
Deleted: SwiftApps/modis/demo5
===================================================================
--- SwiftApps/modis/demo5 2012-01-06 21:39:41 UTC (rev 5488)
+++ SwiftApps/modis/demo5 2012-01-07 07:48:36 UTC (rev 5489)
@@ -1,18 +0,0 @@
-# demo4 forest 50 5 pbs
-
-landtype=${1:-urban}
-nfiles=${2:-50}
-nselect=${3:-10}
-site=${4:-local}
-
-runid=modis-$(date +%Y.%m%d.%H%M)-$landtype-$nfiles-$nselect
-
-echo runid=$runid
-
-swift -tc.file tc.$site \
- -sites.file sites.xml \
- modis5.swift -landtype=$landtype -nfiles=$nfiles -nselect=$nselect -runid=$runid \
- -MODISdir=/home/wilde/bigdata/data/modis/2002 -webdir=/home/wilde/public_html/geo
-exit
-
-swift -tc.file tc.local -sites.file sites.local.xml modis2.swift -landtype=$1 -nfiles=$2 -nselect=$3
Deleted: SwiftApps/modis/demo5.local
===================================================================
--- SwiftApps/modis/demo5.local 2012-01-06 21:39:41 UTC (rev 5488)
+++ SwiftApps/modis/demo5.local 2012-01-07 07:48:36 UTC (rev 5489)
@@ -1,15 +0,0 @@
-# demo4 forest 50 5 pbs
-
-landtype=${1:-urban}
-nfiles=${2:-999}
-nselect=${3:-10}
-site=${4:-local}
-
-runid=modis-$(date +%Y.%m%d.%H%M)-$landtype-$nfiles-$nselect
-
-echo runid=$runid
-
-swift -tc.file tc.local \
- -sites.file sites.local.xml \
- modis5.swift -landtype=$landtype -nfiles=$nfiles -nselect=$nselect -runid=$runid \
- -modisdir=$PWD/data/modis/2002 -webdir=$HOME/public_html/geo
Copied: SwiftApps/modis/modis.swift (from rev 5488, SwiftApps/modis/modis5.swift)
===================================================================
--- SwiftApps/modis/modis.swift (rev 0)
+++ SwiftApps/modis/modis.swift 2012-01-07 07:48:36 UTC (rev 5489)
@@ -0,0 +1,93 @@
+type file;
+type imagefile;
+type landuse;
+
+# Define application program interfaces
+
+app (landuse output) getLandUse (imagefile input, int sortfield)
+{
+ getlanduse @input sortfield stdout=@output ;
+}
+
+app (file output, file tilelist) analyzeLandUse
+ (landuse input[], string usetype, int maxnum)
+{
+ analyzelanduse @output @tilelist usetype maxnum @filenames(input);
+}
+
+app (imagefile output) colorMODIS (imagefile input)
+{
+ colormodis @input @output;
+}
+
+app (imagefile output) assemble
+ (file selected, imagefile image[], string webdir)
+{
+ assemble @output @selected @filename(image[0]) webdir;
+}
+
+app (imagefile grid) markMap (file tilelist)
+{
+ markmap @tilelist @grid;
+}
+
+# Constants and command line arguments
+
+int nFiles = @toint(@arg("nfiles","1000"));
+int nSelect = @toint(@arg("nselect","12"));
+string landType = @arg("landtype","urban");
+string runID = @arg("runid","modis-run");
+string MODISdir= @arg("modisdir","/home/wilde/bigdata/data/modis/2002");
+string webDir = @arg("webdir","/home/wilde/public_html/geo/");
+
+string suffix=".tif";
+
+# Input Dataset
+
+imagefile geos[]
; # site=site
+
+# Compute the land use summary of each MODIS tile
+
+landuse land[] ;
+
+foreach g,i in geos {
+ land[i] = getLandUse(g,1);
+}
+
+# Find the top N tiles (by total area of selected landuse types)
+
+file topSelected<"topselected.txt">;
+file selectedTiles<"selectedtiles.txt">;
+(topSelected, selectedTiles) = analyzeLandUse(land, landType, nSelect);
+
+# Mark the top N tiles on a sinusoidal gridded map
+
+imagefile gridMap<"markedGrid.gif">;
+gridMap = markMap(topSelected);
+
+# Create multi-color images for all tiles
+
+imagefile colorImage[] ;
+
+foreach g, i in geos {
+ colorImage[i] = colorMODIS(g);
+}
+
+# Assemble a montage of the top selected areas
+
+imagefile montage ; # @arg
+montage = assemble(selectedTiles,colorImage,webDir);
+
+# future args:
+
+int selectHiThreshold;
+int selectLowThreshold;
+string upperLeftTile;
+string lowerRightTile;
+float imageSizes[];
+string displayOptions;
+
Deleted: SwiftApps/modis/modis5.swift
===================================================================
--- SwiftApps/modis/modis5.swift 2012-01-06 21:39:41 UTC (rev 5488)
+++ SwiftApps/modis/modis5.swift 2012-01-07 07:48:36 UTC (rev 5489)
@@ -1,93 +0,0 @@
-type file;
-type imagefile;
-type landuse;
-
-# Define application program interfaces
-
-app (landuse output) getLandUse (imagefile input, int sortfield)
-{
- getlanduse @input sortfield stdout=@output ;
-}
-
-app (file output, file tilelist) analyzeLandUse
- (landuse input[], string usetype, int maxnum)
-{
- analyzelanduse @output @tilelist usetype maxnum @filenames(input);
-}
-
-app (imagefile output) colorMODIS (imagefile input)
-{
- colormodis @input @output;
-}
-
-app (imagefile output) assemble
- (file selected, imagefile image[], string webdir)
-{
- assemble @output @selected @filename(image[0]) webdir;
-}
-
-app (imagefile grid) markMap (file tilelist)
-{
- markmap @tilelist @grid;
-}
-
-# Constants and command line arguments
-
-int nFiles = @toint(@arg("nfiles","1000"));
-int nSelect = @toint(@arg("nselect","12"));
-string landType = @arg("landtype","urban");
-string runID = @arg("runid","modis-run");
-string MODISdir= @arg("modisdir","/home/wilde/bigdata/data/modis/2002");
-string webDir = @arg("webdir","/home/wilde/public_html/geo/");
-
-string suffix=".tif";
-
-# Input Dataset
-
-imagefile geos[] ; # site=site
-
-# Compute the land use summary of each MODIS tile
-
-landuse land[] ;
-
-foreach g,i in geos {
- land[i] = getLandUse(g,1);
-}
-
-# Find the top N tiles (by total area of selected landuse types)
-
-file topSelected<"topselected.txt">;
-file selectedTiles<"selectedtiles.txt">;
-(topSelected, selectedTiles) = analyzeLandUse(land, landType, nSelect);
-
-# Mark the top N tiles on a sinusoidal gridded map
-
-imagefile gridMap<"markedGrid.gif">;
-gridMap = markMap(topSelected);
-
-# Create multi-color images for all tiles
-
-imagefile colorImage[] ;
-
-foreach g, i in geos {
- colorImage[i] = colorMODIS(g);
-}
-
-# Assemble a montage of the top selected areas
-
-imagefile montage ; # @arg
-montage = assemble(selectedTiles,colorImage,webDir);
-
-# future args:
-
-int selectHiThreshold;
-int selectLowThreshold;
-string upperLeftTile;
-string lowerRightTile;
-float imageSizes[];
-string displayOptions;
-
Deleted: SwiftApps/modis/selectedtiles.txt
===================================================================
--- SwiftApps/modis/selectedtiles.txt 2012-01-06 21:39:41 UTC (rev 5488)
+++ SwiftApps/modis/selectedtiles.txt 2012-01-07 07:48:36 UTC (rev 5489)
@@ -1,10 +0,0 @@
-/home/wilde/bigdata/data/modis/2002//h30v07.tif
-/home/wilde/bigdata/data/modis/2002//h08v04.tif
-/home/wilde/bigdata/data/modis/2002//h20v12.tif
-/home/wilde/bigdata/data/modis/2002//h19v12.tif
-/home/wilde/bigdata/data/modis/2002//h13v13.tif
-/home/wilde/bigdata/data/modis/2002//h21v10.tif
-/home/wilde/bigdata/data/modis/2002//h10v10.tif
-/home/wilde/bigdata/data/modis/2002//h08v06.tif
-/home/wilde/bigdata/data/modis/2002//h10v08.tif
-/home/wilde/bigdata/data/modis/2002//h24v05.tif
Added: SwiftApps/modis/sites.pads.xml
===================================================================
--- SwiftApps/modis/sites.pads.xml (rev 0)
+++ SwiftApps/modis/sites.pads.xml 2012-01-07 07:48:36 UTC (rev 5489)
@@ -0,0 +1,16 @@
+
+
+
+
+ /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/swiftwork
+ 3600
+ 1
+ 64
+ 1
+ 1
+ fast
+ .6399
+ 10000
+ KEEP
+
+
Deleted: SwiftApps/modis/tc
===================================================================
--- SwiftApps/modis/tc 2012-01-06 21:39:41 UTC (rev 5488)
+++ SwiftApps/modis/tc 2012-01-07 07:48:36 UTC (rev 5489)
@@ -1,34 +0,0 @@
-#This is the transformation catalog.
-#
-#It comes pre-configured with a number of simple transformations with
-#paths that are likely to work on a linux box. However, on some systems,
-#the paths to these executables will be different (for example, sometimes
-#some of these programs are found in /usr/bin rather than in /bin)
-#
-#NOTE WELL: fields in this file must be separated by tabs, not spaces; and
-#there must be no trailing whitespace at the end of each line.
-#
-# site transformation path obsolete fields for compatibility
-
-localhost echo /bin/echo null null null
-localhost cat /bin/cat null null null
-localhost ls /bin/ls null null null
-localhost grep /bin/grep null null null
-localhost sort /bin/sort null null null
-localhost paste /bin/paste null null null
-localhost pwd /bin/pwd null null null
-
-# For cluster usage
-
-pbs convert /usr/bin/convert null null null
-pbs getlanduse /home/wilde/swift/demo/modis/bin/getlanduse.sh null null null
-pbs analyzelanduse /home/wilde/swift/demo/modis/bin/analyzelandusep1.sh null null null
-pbs colormodis /home/wilde/swift/demo/modis/bin/colormodis.sh null null null
-pbs assemble /home/wilde/swift/demo/modis/bin/assemble.sh null null null
-
-# For localhost testing
-
-#localhost convert /usr/bin/convert null null null
-#localhost getlanduse /home/wilde/bigdata/bin/getlanduse.sh null null null
-#localhost analyzelanduse /home/wilde/bigdata/bin/analyzelanduse.sh null null null
-#localhost colormodis /home/wilde/bigdata/bin/colormodis.sh null null null
Added: SwiftApps/modis/tc.pads
===================================================================
--- SwiftApps/modis/tc.pads (rev 0)
+++ SwiftApps/modis/tc.pads 2012-01-07 07:48:36 UTC (rev 5489)
@@ -0,0 +1,26 @@
+#This is the transformation catalog.
+#
+#It comes pre-configured with a number of simple transformations with
+#paths that are likely to work on a linux box. However, on some systems,
+#the paths to these executables will be different (for example, sometimes
+#some of these programs are found in /usr/bin rather than in /bin)
+#
+#NOTE WELL: fields in this file must be separated by tabs, not spaces; and
+#there must be no trailing whitespace at the end of each line.
+#
+# site transformation path obsolete fields for compatibility
+
+localhost echo /bin/echo null null null
+localhost cat /bin/cat null null null
+localhost ls /bin/ls null null null
+localhost grep /bin/grep null null null
+localhost sort /bin/sort null null null
+localhost paste /bin/paste null null null
+localhost pwd /bin/pwd null null null
+
+# For cluster usage
+pads getlanduse /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/modis/bin/getlanduse.sh null null null
+pads analyzelanduse /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/modis/bin/analyzelanduse2.sh null null null
+pads colormodis /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/modis/bin/colormodis.sh null null null
+pads assemble /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/modis/bin/assemble2.sh null null null
+pads markmap /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/modis/bin/markmap.sh null null null
From davidk at ci.uchicago.edu Mon Jan 9 09:58:22 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Mon, 9 Jan 2012 09:58:22 -0600 (CST)
Subject: [Swift-commit] r5490 - SwiftApps/modis
Message-ID: <20120109155822.A76339CD18@svn.ci.uchicago.edu>
Author: davidk
Date: 2012-01-09 09:58:22 -0600 (Mon, 09 Jan 2012)
New Revision: 5490
Added:
SwiftApps/modis/demo.pads-pbs
SwiftApps/modis/sites.pads-pbs.xml
Removed:
SwiftApps/modis/topselected.txt
Log:
Config files for PADS with plain pbs provider
Added: SwiftApps/modis/demo.pads-pbs
===================================================================
--- SwiftApps/modis/demo.pads-pbs (rev 0)
+++ SwiftApps/modis/demo.pads-pbs 2012-01-09 15:58:22 UTC (rev 5490)
@@ -0,0 +1,15 @@
+# demo4 forest 50 5 pbs
+
+landtype=${1:-urban}
+nfiles=${2:-999}
+nselect=${3:-10}
+site=${4:-local}
+
+runid=modis-$(date +%Y.%m%d.%H%M)-$landtype-$nfiles-$nselect
+
+echo runid=$runid
+
+swift -tc.file tc.pads \
+ -sites.file sites.pads-pbs.xml \
+ modis.swift -config cf.pads -landtype=$landtype -nfiles=$nfiles -nselect=$nselect -runid=$runid \
+ -modisdir=$PWD/data/modis/2002 -webdir=$HOME/public_html/geo
Property changes on: SwiftApps/modis/demo.pads-pbs
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/modis/sites.pads-pbs.xml
===================================================================
--- SwiftApps/modis/sites.pads-pbs.xml (rev 0)
+++ SwiftApps/modis/sites.pads-pbs.xml 2012-01-09 15:58:22 UTC (rev 5490)
@@ -0,0 +1,16 @@
+
+
+
+
+ /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/swiftwork
+ 3600
+ 1
+ 64
+ 1
+ 1
+ fast
+ .6399
+ 10000
+ KEEP
+
+
Deleted: SwiftApps/modis/topselected.txt
===================================================================
--- SwiftApps/modis/topselected.txt 2012-01-07 07:48:36 UTC (rev 5489)
+++ SwiftApps/modis/topselected.txt 2012-01-09 15:58:22 UTC (rev 5490)
@@ -1,10 +0,0 @@
-modis-2011.1201.1740-urban-30-10/h30v07.landuse.byfreq 1864
-modis-2011.1201.1740-urban-30-10/h08v04.landuse.byfreq 2541
-modis-2011.1201.1740-urban-30-10/h20v12.landuse.byfreq 3308
-modis-2011.1201.1740-urban-30-10/h19v12.landuse.byfreq 4283
-modis-2011.1201.1740-urban-30-10/h13v13.landuse.byfreq 5022
-modis-2011.1201.1740-urban-30-10/h21v10.landuse.byfreq 5370
-modis-2011.1201.1740-urban-30-10/h10v10.landuse.byfreq 8663
-modis-2011.1201.1740-urban-30-10/h08v06.landuse.byfreq 14824
-modis-2011.1201.1740-urban-30-10/h10v08.landuse.byfreq 19487
-modis-2011.1201.1740-urban-30-10/h24v05.landuse.byfreq 25405
From tga at ci.uchicago.edu Mon Jan 9 11:06:36 2012
From: tga at ci.uchicago.edu (tga at ci.uchicago.edu)
Date: Mon, 9 Jan 2012 11:06:36 -0600 (CST)
Subject: [Swift-commit] r5491 - SwiftApps/SwiftR
Message-ID: <20120109170636.9AD4A9CD18@svn.ci.uchicago.edu>
Author: tga
Date: 2012-01-09 11:06:36 -0600 (Mon, 09 Jan 2012)
New Revision: 5491
Modified:
SwiftApps/SwiftR/checkout-swift.sh
Log:
remove trace from install shell script
Modified: SwiftApps/SwiftR/checkout-swift.sh
===================================================================
--- SwiftApps/SwiftR/checkout-swift.sh 2012-01-09 15:58:22 UTC (rev 5490)
+++ SwiftApps/SwiftR/checkout-swift.sh 2012-01-09 17:06:36 UTC (rev 5491)
@@ -1,6 +1,5 @@
#!/bin/sh
set -e
-set -x
COG_SRC=$1
SWIFT_PATCH=$2
SWIFT_TAG=$3
From tga at ci.uchicago.edu Mon Jan 9 11:06:38 2012
From: tga at ci.uchicago.edu (tga at ci.uchicago.edu)
Date: Mon, 9 Jan 2012 11:06:38 -0600 (CST)
Subject: [Swift-commit] r5492 - SwiftApps/SwiftR/Swift/src
Message-ID: <20120109170638.1DD9E9CD18@svn.ci.uchicago.edu>
Author: tga
Date: 2012-01-09 11:06:38 -0600 (Mon, 09 Jan 2012)
New Revision: 5492
Modified:
SwiftApps/SwiftR/Swift/src/make.include
Log:
Switch to using 0.93 swift release
Modified: SwiftApps/SwiftR/Swift/src/make.include
===================================================================
--- SwiftApps/SwiftR/Swift/src/make.include 2012-01-09 17:06:36 UTC (rev 5491)
+++ SwiftApps/SwiftR/Swift/src/make.include 2012-01-09 17:06:38 UTC (rev 5492)
@@ -1,13 +1,18 @@
# Variables shared between makefiles
# Use 0.92.1 release
-SWIFT_SRC_TAG = release-0.92.1
-COG_SRC_TAG = swift_0.92.1
-SWIFT_SRC_NAME = swift-0.92.1-SwiftR-r2
+#SWIFT_SRC_TAG = tags/release-0.92.1
+#COG_SRC_TAG = tags/swift_0.92.1
+#SWIFT_SRC_NAME = swift-0.92.1-SwiftR-r2
-# Use 0.93 release candidate branch
+# Use 0.93 release candidate
#SWIFT_SRC_TAG = branches/release-0.93
#COG_SRC_TAG = branches/4.1.9/src/cog
#SWIFT_SRC_NAME = swift-0.93-SwiftR-r1
+# Use 0.93 release
+SWIFT_SRC_TAG = tags/release-0.93
+COG_SRC_TAG = tags/swift-0.93/src/cog
+SWIFT_SRC_NAME = swift-0.93-SwiftR-r2
+
SWIFT_SRC_PATCH = swift-0.92.1-changes.patch
From davidk at ci.uchicago.edu Mon Jan 9 11:44:44 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Mon, 9 Jan 2012 11:44:44 -0600 (CST)
Subject: [Swift-commit] r5493 - SwiftApps/modis
Message-ID: <20120109174444.474D59CD18@svn.ci.uchicago.edu>
Author: davidk
Date: 2012-01-09 11:44:44 -0600 (Mon, 09 Jan 2012)
New Revision: 5493
Added:
SwiftApps/modis/README
Log:
Add readme file for some instructions on how to run
Added: SwiftApps/modis/README
===================================================================
--- SwiftApps/modis/README (rev 0)
+++ SwiftApps/modis/README 2012-01-09 17:44:44 UTC (rev 5493)
@@ -0,0 +1,51 @@
+demo.local
+----------
+This script will run the modis demo using only the processors available on the
+local machine. Modify tc.local to reflect the path to the modis applications.
+Run the demo by running:
+
+./demo.local
+
+This will use all 317 input files. To limit the number of task that will be
+created, run:
+
+./demo.local urban
+
+
+demo.pads
+---------
+This script will run the demo on PADS using coasters. This should be run
+from a PADS login node (login.pads.ci.uchicago.edu)
+
+Edit the 'swift' application and modify the line
+that says: HEAPMAX=256M. Change it to HEAPMAX=1024M.
+
+Edit tc.pads to reflect the path to the modis applications.
+
+Make sure that your default PADS project is defined. You can check this
+by running the 'projects' command.
+
+Run the demo with the command:
+
+nice ./demo.pads
+
+This will use all 317 input files. to limit the number of tasks that will be
+created, run:
+
+nice ./demo.local urban
+
+
+demo.pads-pbs
+-------------
+The instructions for running this are the same as for demo.pads. The only
+difference between the two is that demo.pads uses coasters while
+demo.pads-pbs uses the plain PBS provider.
+
+nice ./demo.pads-pbs
+
+
+Output
+------
+When the script is completed, you should see several output files created in
+the landuse directory. Another graphic will be created in the directory
+$HOME/public_html/geo.
From lgadelha at ci.uchicago.edu Mon Jan 9 12:29:07 2012
From: lgadelha at ci.uchicago.edu (lgadelha at ci.uchicago.edu)
Date: Mon, 9 Jan 2012 12:29:07 -0600 (CST)
Subject: [Swift-commit] r5494 - provenancedb
Message-ID: <20120109182907.870F29CD18@svn.ci.uchicago.edu>
Author: lgadelha
Date: 2012-01-09 12:29:07 -0600 (Mon, 09 Jan 2012)
New Revision: 5494
Modified:
provenancedb/prov-init.sql
Log:
Minor changes
Modified: provenancedb/prov-init.sql
===================================================================
--- provenancedb/prov-init.sql 2012-01-09 17:44:44 UTC (rev 5493)
+++ provenancedb/prov-init.sql 2012-01-09 18:29:07 UTC (rev 5494)
@@ -195,6 +195,15 @@
q varchar(2048)
);
+create view function as select * from proc natural join app_inv;
+
+create view dataset as
+ select file.id, 'mapped' as type, file.name as filename, null as value
+ from file
+ union
+ select in_mem.id, 'primitive' as type, null as filename, in_mem.value
+ from in_mem;
+
drop view pgraph_edge;
create view pgraph_edge as
select proc_id as parent,ds_id as child from ds_out
From tga at ci.uchicago.edu Mon Jan 9 22:05:20 2012
From: tga at ci.uchicago.edu (tga at ci.uchicago.edu)
Date: Mon, 9 Jan 2012 22:05:20 -0600 (CST)
Subject: [Swift-commit] r5495 - SwiftApps/SwiftR/Swift/exec
Message-ID: <20120110040521.01C349CD18@svn.ci.uchicago.edu>
Author: tga
Date: 2012-01-09 22:05:20 -0600 (Mon, 09 Jan 2012)
New Revision: 5495
Modified:
SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh
Log:
Retain user umask
Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh
===================================================================
--- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2012-01-09 18:29:07 UTC (rev 5494)
+++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2012-01-10 04:05:20 UTC (rev 5495)
@@ -7,7 +7,9 @@
# Set restrictive umask for duration of script
# This value prevents any other users from reading
# or writing
-umask "077"
+DEFAULT_UMASK=077
+USER_UMASK=$(umask)
+umask $DEFAULT_UMASK
#if [ $# != 4 ]; then
if [ $# -lt 4 ]; then
@@ -90,10 +92,13 @@
echo R_LIBS_USER= $R_LIBS_USER
slotdir=$1
+
# Use R instead of Rscript due to Rscript issues on some
# platforms (e.g. cray xt)
+ umask $USER_UMASK
R --slave --no-restore --file=$RServerScript \
--args $slotdir >> $slotdir/R.log 2>&1 & # launch R server
+ umask $DEFAULT_UMASK
RPID=$!
echo $RPID > $SLOTDIR/R.pid
echo "$0: INFO: Launched $RServerScript $slotdir Rscript"
From davidk at ci.uchicago.edu Tue Jan 10 11:01:34 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Tue, 10 Jan 2012 11:01:34 -0600 (CST)
Subject: [Swift-commit] r5496 - in trunk/tests/sites: . ibicluster
Message-ID: <20120110170134.81DE69CC42@svn.ci.uchicago.edu>
Author: davidk
Date: 2012-01-10 11:01:34 -0600 (Tue, 10 Jan 2012)
New Revision: 5496
Added:
trunk/tests/sites/ibicluster/
trunk/tests/sites/ibicluster/catsn.0001.out.expected
trunk/tests/sites/ibicluster/catsn.0002.out.expected
trunk/tests/sites/ibicluster/catsn.0003.out.expected
trunk/tests/sites/ibicluster/catsn.0004.out.expected
trunk/tests/sites/ibicluster/catsn.0005.out.expected
trunk/tests/sites/ibicluster/catsn.0006.out.expected
trunk/tests/sites/ibicluster/catsn.0007.out.expected
trunk/tests/sites/ibicluster/catsn.0008.out.expected
trunk/tests/sites/ibicluster/catsn.0009.out.expected
trunk/tests/sites/ibicluster/catsn.0010.out.expected
trunk/tests/sites/ibicluster/catsn.check.sh
trunk/tests/sites/ibicluster/catsn.setup.sh
trunk/tests/sites/ibicluster/catsn.swift
trunk/tests/sites/ibicluster/catsn.timeout
trunk/tests/sites/ibicluster/data.txt
trunk/tests/sites/ibicluster/sites.template.xml
trunk/tests/sites/ibicluster/tc.template.data
trunk/tests/sites/ibicluster/title.txt
Log:
Add site test for ibicluster
Added: trunk/tests/sites/ibicluster/catsn.0001.out.expected
===================================================================
--- trunk/tests/sites/ibicluster/catsn.0001.out.expected (rev 0)
+++ trunk/tests/sites/ibicluster/catsn.0001.out.expected 2012-01-10 17:01:34 UTC (rev 5496)
@@ -0,0 +1 @@
+blah blah
Added: trunk/tests/sites/ibicluster/catsn.0002.out.expected
===================================================================
--- trunk/tests/sites/ibicluster/catsn.0002.out.expected (rev 0)
+++ trunk/tests/sites/ibicluster/catsn.0002.out.expected 2012-01-10 17:01:34 UTC (rev 5496)
@@ -0,0 +1 @@
+blah blah
Added: trunk/tests/sites/ibicluster/catsn.0003.out.expected
===================================================================
--- trunk/tests/sites/ibicluster/catsn.0003.out.expected (rev 0)
+++ trunk/tests/sites/ibicluster/catsn.0003.out.expected 2012-01-10 17:01:34 UTC (rev 5496)
@@ -0,0 +1 @@
+blah blah
Added: trunk/tests/sites/ibicluster/catsn.0004.out.expected
===================================================================
--- trunk/tests/sites/ibicluster/catsn.0004.out.expected (rev 0)
+++ trunk/tests/sites/ibicluster/catsn.0004.out.expected 2012-01-10 17:01:34 UTC (rev 5496)
@@ -0,0 +1 @@
+blah blah
Added: trunk/tests/sites/ibicluster/catsn.0005.out.expected
===================================================================
--- trunk/tests/sites/ibicluster/catsn.0005.out.expected (rev 0)
+++ trunk/tests/sites/ibicluster/catsn.0005.out.expected 2012-01-10 17:01:34 UTC (rev 5496)
@@ -0,0 +1 @@
+blah blah
Added: trunk/tests/sites/ibicluster/catsn.0006.out.expected
===================================================================
--- trunk/tests/sites/ibicluster/catsn.0006.out.expected (rev 0)
+++ trunk/tests/sites/ibicluster/catsn.0006.out.expected 2012-01-10 17:01:34 UTC (rev 5496)
@@ -0,0 +1 @@
+blah blah
Added: trunk/tests/sites/ibicluster/catsn.0007.out.expected
===================================================================
--- trunk/tests/sites/ibicluster/catsn.0007.out.expected (rev 0)
+++ trunk/tests/sites/ibicluster/catsn.0007.out.expected 2012-01-10 17:01:34 UTC (rev 5496)
@@ -0,0 +1 @@
+blah blah
Added: trunk/tests/sites/ibicluster/catsn.0008.out.expected
===================================================================
--- trunk/tests/sites/ibicluster/catsn.0008.out.expected (rev 0)
+++ trunk/tests/sites/ibicluster/catsn.0008.out.expected 2012-01-10 17:01:34 UTC (rev 5496)
@@ -0,0 +1 @@
+blah blah
Added: trunk/tests/sites/ibicluster/catsn.0009.out.expected
===================================================================
--- trunk/tests/sites/ibicluster/catsn.0009.out.expected (rev 0)
+++ trunk/tests/sites/ibicluster/catsn.0009.out.expected 2012-01-10 17:01:34 UTC (rev 5496)
@@ -0,0 +1 @@
+blah blah
Added: trunk/tests/sites/ibicluster/catsn.0010.out.expected
===================================================================
--- trunk/tests/sites/ibicluster/catsn.0010.out.expected (rev 0)
+++ trunk/tests/sites/ibicluster/catsn.0010.out.expected 2012-01-10 17:01:34 UTC (rev 5496)
@@ -0,0 +1 @@
+blah blah
Added: trunk/tests/sites/ibicluster/catsn.check.sh
===================================================================
--- trunk/tests/sites/ibicluster/catsn.check.sh (rev 0)
+++ trunk/tests/sites/ibicluster/catsn.check.sh 2012-01-10 17:01:34 UTC (rev 5496)
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -x
+
+for count in `seq --format "%04.f" 1 1 10`
+do
+ [ -f catsn.$count.out ] || exit 1
+ CONTENTS1=$( cat catsn.$count.out.expected )
+ CONTENTS2=$( cat catsn.$count.out )
+ [[ $CONTENTS1 == $CONTENTS2 ]] || exit 1
+done
+exit 0
Property changes on: trunk/tests/sites/ibicluster/catsn.check.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: trunk/tests/sites/ibicluster/catsn.setup.sh
===================================================================
--- trunk/tests/sites/ibicluster/catsn.setup.sh (rev 0)
+++ trunk/tests/sites/ibicluster/catsn.setup.sh 2012-01-10 17:01:34 UTC (rev 5496)
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cp -v $GROUP/data.txt . || exit 1
+cp -v $GROUP/*expected . || exit 1
+
+export QUEUE=normal
+
Property changes on: trunk/tests/sites/ibicluster/catsn.setup.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: trunk/tests/sites/ibicluster/catsn.swift
===================================================================
--- trunk/tests/sites/ibicluster/catsn.swift (rev 0)
+++ trunk/tests/sites/ibicluster/catsn.swift 2012-01-10 17:01:34 UTC (rev 5496)
@@ -0,0 +1,12 @@
+type file;
+
+app (file o) cat (file i)
+{
+ cat @i stdout=@o;
+}
+
+file out[];
+foreach j in [1:@toint(@arg("n","10"))] {
+ file data<"data.txt">;
+ out[j] = cat(data);
+}
Added: trunk/tests/sites/ibicluster/catsn.timeout
===================================================================
--- trunk/tests/sites/ibicluster/catsn.timeout (rev 0)
+++ trunk/tests/sites/ibicluster/catsn.timeout 2012-01-10 17:01:34 UTC (rev 5496)
@@ -0,0 +1 @@
+1800
Added: trunk/tests/sites/ibicluster/data.txt
===================================================================
--- trunk/tests/sites/ibicluster/data.txt (rev 0)
+++ trunk/tests/sites/ibicluster/data.txt 2012-01-10 17:01:34 UTC (rev 5496)
@@ -0,0 +1 @@
+blah blah
Added: trunk/tests/sites/ibicluster/sites.template.xml
===================================================================
--- trunk/tests/sites/ibicluster/sites.template.xml (rev 0)
+++ trunk/tests/sites/ibicluster/sites.template.xml 2012-01-10 17:01:34 UTC (rev 5496)
@@ -0,0 +1,14 @@
+
+
+
+ threaded
+ all.q
+ 8
+ 1
+ 2
+ 0.159
+ 10000
+
+ _WORK_
+
+
Added: trunk/tests/sites/ibicluster/tc.template.data
===================================================================
--- trunk/tests/sites/ibicluster/tc.template.data (rev 0)
+++ trunk/tests/sites/ibicluster/tc.template.data 2012-01-10 17:01:34 UTC (rev 5496)
@@ -0,0 +1,8 @@
+ibicluster echo /bin/echo INSTALLED INTEL32::LINUX
+ibicluster cat /bin/cat INSTALLED INTEL32::LINUX
+ibicluster ls /bin/ls INSTALLED INTEL32::LINUX
+ibicluster grep /bin/grep INSTALLED INTEL32::LINUX
+ibicluster sort /bin/sort INSTALLED INTEL32::LINUX
+ibicluster paste /bin/paste INSTALLED INTEL32::LINUX
+ibicluster wc /usr/bin/wc INSTALLED INTEL32::LINUX
+
Added: trunk/tests/sites/ibicluster/title.txt
===================================================================
--- trunk/tests/sites/ibicluster/title.txt (rev 0)
+++ trunk/tests/sites/ibicluster/title.txt 2012-01-10 17:01:34 UTC (rev 5496)
@@ -0,0 +1 @@
+IBICluster SGE Test
From tga at ci.uchicago.edu Tue Jan 10 14:21:46 2012
From: tga at ci.uchicago.edu (tga at ci.uchicago.edu)
Date: Tue, 10 Jan 2012 14:21:46 -0600 (CST)
Subject: [Swift-commit] r5498 - SwiftApps/SwiftR/Swift/exec
Message-ID: <20120110202146.05E989CCBF@svn.ci.uchicago.edu>
Author: tga
Date: 2012-01-10 14:21:45 -0600 (Tue, 10 Jan 2012)
New Revision: 5498
Modified:
SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh
Log:
Make log message a bit more readable
Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh
===================================================================
--- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2012-01-10 18:43:45 UTC (rev 5497)
+++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2012-01-10 20:21:45 UTC (rev 5498)
@@ -210,8 +210,9 @@
function timeout_handler {
echo "Timed out after ${TIMEOUT}s waiting to contact R process"
echo 'R log follows:'
- echo '====================='
+ echo '>>>>>R.log>>>>>'
cat $SLOTDIR/R.log
+ echo '<<<<
Author: lgadelha
Date: 2012-01-17 10:53:00 -0600 (Tue, 17 Jan 2012)
New Revision: 5499
Modified:
provenancedb/prov-init.sql
Log:
New views for schema summarization
Modified: provenancedb/prov-init.sql
===================================================================
--- provenancedb/prov-init.sql 2012-01-10 20:21:45 UTC (rev 5498)
+++ provenancedb/prov-init.sql 2012-01-17 16:53:00 UTC (rev 5499)
@@ -195,15 +195,7 @@
q varchar(2048)
);
-create view function as select * from proc natural join app_inv;
-create view dataset as
- select file.id, 'mapped' as type, file.name as filename, null as value
- from file
- union
- select in_mem.id, 'primitive' as type, null as filename, in_mem.value
- from in_mem;
-
drop view pgraph_edge;
create view pgraph_edge as
select proc_id as parent,ds_id as child from ds_out
@@ -214,28 +206,81 @@
-- continue renaming from here
-CREATE VIEW run_date AS
- SELECT id, log_filename, swift_version, cog_version, final_state,
- to_timestamp(start_time) as start_time, duration
- FROM run;
CREATE VIEW a_t AS
SELECT *
FROM a_run_t
- UNION
+ UNION ALL
SELECT *
FROM a_ds_t
- UNION
+ UNION ALL
SELECT *
FROM a_proc_t;
CREATE VIEW a_n AS
SELECT *
FROM a_run_n
- UNION
+ UNION ALL
SELECT *
FROM a_ds_n
- UNION
+ UNION ALL
SELECT *
FROM a_proc_n;
+-- views used for queries based on the schema summary
+
+drop view function_call;
+
+create view function_call as
+ select proc.id, proc.type, proc.name, proc.run_id, app_inv.proc_name, to_timestamp(app_inv.start_time), app_inv.duration, app_inv.final_state, app_inv.scratch from proc natural join app_inv;
+
+drop view variable;
+
+create view variable as
+ select file.id, 'mapped' as type, file.name as filename, null as value
+ from file
+ union all
+ select in_mem.id, 'primitive' as type, null as filename, in_mem.value
+ from in_mem
+ union all
+ select ds_cont.out_id as id, 'composite' as type, null as filename, null as value from ds_cont;
+
+drop view annotation;
+
+create view annotation as
+ select a_t.run_id as id, a_t.name as key, a_t.value as string_value, null as numeric_value
+ from a_t
+ union all
+ select a_n.run_id as id, a_n.name as key, null as string_value, a_n.value as numeric_value
+ from a_n;
+
+drop view script_run;
+
+create view script_run as
+ select id, log_filename, swift_version, cog_version, final_state,
+ to_timestamp(start_time) as start_time, duration
+ from run;
+
+
+drop view application_execution;
+
+create view application_execution as
+ select id, app_inv_id as function_call_id, to_timestamp(start_time) as start_time, duration, final_state, site
+ from app_exec;
+
+
+drop view runtime_info;
+
+create view runtime_info as
+ select app_exec_id as application_execution_id, tstamp as timestamp, cpu_usage, max_phys_mem, max_virt_mem, io_read, io_write
+ from rt_info;
+
+drop view produces;
+
+create view produces as
+ select proc_id as function_call_id, ds_id as variable_id, param as parameter from ds_out;
+
+drop view consumes;
+
+create view consumes as
+ select proc_id as function_call_id, ds_id as variable_id, param as parameter from ds_in;
From ketan at ci.uchicago.edu Tue Jan 17 11:17:08 2012
From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu)
Date: Tue, 17 Jan 2012 11:17:08 -0600 (CST)
Subject: [Swift-commit] r5500 - trunk/docs/siteguide
Message-ID: <20120117171708.6E2A09CC94@svn.ci.uchicago.edu>
Author: ketan
Date: 2012-01-17 11:17:08 -0600 (Tue, 17 Jan 2012)
New Revision: 5500
Modified:
trunk/docs/siteguide/grid
Log:
siteguide refinements
Modified: trunk/docs/siteguide/grid
===================================================================
--- trunk/docs/siteguide/grid 2012-01-17 16:53:00 UTC (rev 5499)
+++ trunk/docs/siteguide/grid 2012-01-17 17:17:08 UTC (rev 5500)
@@ -33,7 +33,7 @@
~~~~~~~~~~~~~~~~~
*For OSG*: Obtain a DOEGrids certificate and register the certificate in
-the OSG "Engage" VO following the procedures at:
+the OSG "Engage" VO following the procedure at:
https://twiki.grid.iu.edu/bin/view/Engagement/EngageNewUserGuide
@@ -91,14 +91,14 @@
$ voms-proxy-init --voms Engage -hours 48
----
-To run jobs using the procedures documented here you need to be logged
+To run jobs using the procedure described here, you need to be logged
in to a "submit host" on which you will run Swift and other
-grid-related utilities. This can be any host with the OSG client stack
-or equivalent tools installed. Such hosts include the OSG Engage
-submit host, and the two Swift lab servers
+grid-related utilities. A submit host is any host where the OSG client stack
+or equivalent tools are installed. Such hosts include the OSG Engage
+submit host (engage-submit3.renci.org), and the two Swift lab servers
{bridled,communicado}.ci.uchicago.edu.
-Obtain a login on engage-submit.renci.org following instructions on
+Obtain a login on engage-submit3.renci.org following instructions on
the OSG URL above.
Obtain a CI login with access to the Swift lab servers by requesting
From ketan at ci.uchicago.edu Tue Jan 17 13:44:37 2012
From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu)
Date: Tue, 17 Jan 2012 13:44:37 -0600 (CST)
Subject: [Swift-commit] r5501 - trunk/docs/siteguide
Message-ID: <20120117194437.11CD39CC94@svn.ci.uchicago.edu>
Author: ketan
Date: 2012-01-17 13:44:36 -0600 (Tue, 17 Jan 2012)
New Revision: 5501
Modified:
trunk/docs/siteguide/grid
Log:
siteguide refinements
Modified: trunk/docs/siteguide/grid
===================================================================
--- trunk/docs/siteguide/grid 2012-01-17 17:17:08 UTC (rev 5500)
+++ trunk/docs/siteguide/grid 2012-01-17 19:44:36 UTC (rev 5501)
@@ -138,16 +138,18 @@
Depending on your shell type, run:
-----
-source /opt/osg/setup.sh
+source /opt/osg-/setup.sh
or
-source /opt/osg/setup.csh
+source /opt/osg-/setup.csh
-----
+NOTE: This above step is not required on engage-submit3 host.
+
Create a VOMS Grid proxy
~~~~~~~~~~~~~~~~~~~~~~~~
-----
-$ voms-proxy-init -voms Engage -valid 12:00
+$ voms-proxy-init -voms Engage -valid 72:00
-----
Generating Configuration Files
@@ -204,12 +206,37 @@
$
$ foreachsite -resource fork myinstall.sh
$
-$ # Wait a file here, then poll for successfully installed apps...
+$ # Wait a while here, then poll for successfully installed apps...
$
$ grep SUCCEEDED run.89/*/*.stdout
$
-----
+Following is an example of the installation script for DSSAT app on OSG:
+
+-----
+#!/bin/bash
+
+cd ${OSG_APP}/extenci/swift/
+
+#pull
+wget http://www.ci.uchicago.edu/~ketan/DSSAT.tgz
+
+#extract
+tar zxf DSSAT.tgz
+
+# test
+cd DSSAT/data
+
+../DSSAT040.EXE A H1234567.MZX > std.OUT
+
+if [ $? = 0 ]; then
+ echo INSTALL SUCCEEDED
+else
+ echo INSTALL FAILED
+fi
+-----
+
Starting a single coaster service
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
From wilde at ci.uchicago.edu Fri Jan 20 15:32:59 2012
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Fri, 20 Jan 2012 15:32:59 -0600 (CST)
Subject: [Swift-commit] r5503 - SwiftApps
Message-ID: <20120120213259.E42B69CD00@svn.ci.uchicago.edu>
Author: wilde
Date: 2012-01-20 15:32:59 -0600 (Fri, 20 Jan 2012)
New Revision: 5503
Added:
SwiftApps/SciColSim/
Log:
New dir for scientific collaboration network simulator.
From hategan at ci.uchicago.edu Sat Jan 21 21:29:59 2012
From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu)
Date: Sat, 21 Jan 2012 21:29:59 -0600 (CST)
Subject: [Swift-commit] r5506 - branches/release-0.93.1/tests
Message-ID: <20120122032959.9F22A9CC94@svn.ci.uchicago.edu>
Author: hategan
Date: 2012-01-21 21:29:59 -0600 (Sat, 21 Jan 2012)
New Revision: 5506
Added:
branches/release-0.93.1/tests/providers
Log:
added providers directory (symlink to sites)
Added: branches/release-0.93.1/tests/providers
===================================================================
--- branches/release-0.93.1/tests/providers (rev 0)
+++ branches/release-0.93.1/tests/providers 2012-01-22 03:29:59 UTC (rev 5506)
@@ -0,0 +1 @@
+link sites
\ No newline at end of file
Property changes on: branches/release-0.93.1/tests/providers
___________________________________________________________________
Added: svn:special
+ *
From wilde at ci.uchicago.edu Sun Jan 22 19:50:27 2012
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Sun, 22 Jan 2012 19:50:27 -0600 (CST)
Subject: [Swift-commit] r5507 - SwiftApps/SciColSim
Message-ID: <20120123015027.BBFB09CCA8@svn.ci.uchicago.edu>
Author: wilde
Date: 2012-01-22 19:50:27 -0600 (Sun, 22 Jan 2012)
New Revision: 5507
Modified:
SwiftApps/SciColSim/annealing.swift
SwiftApps/SciColSim/math.swift
SwiftApps/SciColSim/tc
Log:
Initial working version of basic swift annealing loop.
Modified: SwiftApps/SciColSim/annealing.swift
===================================================================
--- SwiftApps/SciColSim/annealing.swift 2012-01-22 03:29:59 UTC (rev 5506)
+++ SwiftApps/SciColSim/annealing.swift 2012-01-23 01:50:27 UTC (rev 5507)
@@ -19,6 +19,7 @@
} else {
nx = x - (random())*dx/(pow(2.0,31.0)-1.0);
}
+ tracef("newx(%f,%f)=%f\n",x,dx,nx);
}
app (file outfile, file loss) evolve (string args[], file graph)
@@ -40,7 +41,7 @@
multi_annealing()
multi_loss()
evolve()
- sumloass()
+ sumloss()
*/
(file bestfile, file maxfile) multi_annealing (
@@ -61,7 +62,8 @@
float x[][], dx[][], curr_loss[], curr_sdev[];
Res mlres[][];
- mlres[0][0] = multi_loss( params0, target_innov, evolve_reruns ); // Only done once, not 5x; serves for all evolve-params ???
+ mlres[0][0] = multi_loss( 0, 0, params0, target_innov, evolve_reruns ); // Only done once, not 5x; serves for all evolve-params ???
+tracef("in multi_annealing: i=%i j=%i ret vals from iniital multi_loss=(%f,%f)\n",0,0,mlres[0][0].loss,mlres[0][0].sdev);
foreach j in [0:NEVOPARAMS-1] {
x[0][j]=params0[j];
@@ -74,9 +76,10 @@
foreach i in [1:annealing_cycles] {
// set new temperature, rejection threshold, and dx values for this cycle
float temperature = T_start*exp( @tofloat(i)*(jlog(T_end)-jlog(T_start))/@tofloat(annealing_cycles));
- tracef("....T =%f\n", temperature);
+ tracef("in multi_annealing: i=%i T=%f\n", i, temperature);
// On each new "major" cycle within the annealing_cycles (other than the first) set new rejection and dx values
if ( i %% cycle == 1 && i > 1 ){
+ tracef("multi_annealing: new cycle at i=%i\n",i);
foreach k in [0:NEVOPARAMS-1] {
float newrejection = rejection[i-1][k] / @tofloat(cycle);
if (newrejection > 0.0){
@@ -90,47 +93,61 @@
trace ("Recomputed rejection: i=%d k=%d dx[i][k]=%f\n", i, k, dx[i][k]);
}
}
+ else { # If not new cycle, set dx from previous dx (i-1)
+ foreach k in [0:NEVOPARAMS-1] {
+ dx[i][k] = dx[i-1][k];
+ }
+ }
foreach j in [0:NEVOPARAMS-1] { // Try a new value for each non-fixed param; then write results and accept or reject
- float try_x[];
+ // float try_x[];
int curr = (i * NEVOPARAMS) + j;
int prev = curr-1;
- if ( FIX_VARIABLES || var_fixed[j]==0) {
- foreach k in [0:NEVOPARAMS] { // Select the evolve params to try
+tracef("in multi_annealing: i=%i j=%i curr=%i prev=%i\n", i, j, curr, prev);
+ if ( /*(!FIX_VARIABLES) || */ (var_fixed[j]==0) ) { # fixed=1,1,0,0,0: FIXME: FIX_VARIABLES flag has faulty logic but OK when TRUE
+ float try_x[];
+ foreach k in [0:NEVOPARAMS-1] { // Select the evolve params to try
if ( k < j ) {
try_x[k] = x[i][k]; // already set x[i][k]
}
else {
if ( k == j ) {
- try_x[k] = newx(x[i][j],dx[i][j]); // permute x[i][j]
+ try_x[k] = newx(x[i-1][j],dx[i-1][j]); // permute x[i-1][j]
}
- else {
+ else { // k > j
try_x[k] = x[i-1][k]; // use x[i-1][k] (from prior cycle)
}
}
+ tracef("in multi_annealing: i=%i j=%i k=%i try_x=%f\n", i, j, k, try_x[k]);
+ tracef("in multi_annealing: i=%i j=%i k=%i\n", i, j, k);
}
-
- mlres[i][j] = multi_loss(try_x, target_innov, evolve_reruns);
-
+ // Up to here, x[] and dx[] are only set for previous i
+tracef("in multi_annealing: i=%i j=%i calling multi_loss\n",i,j);
+#tracef("in multi_annealing: i=%i j=%i calling multi_loss try_x=%q\n",i,j,try_x);
+ mlres[i][j] = multi_loss(i,j,try_x, target_innov, evolve_reruns); # do the N evolve()'s, N=evolve_reruns
+tracef("in multi_annealing: i=%i j=%i ret vals from multi_loss=(%f,%f)\n",i,j,mlres[i][j].loss,mlres[i][j].sdev);
+ // Beyond this point, x[] and dx[] are being set for this i,j
float ratio = min(1.0, exp( -(mlres[i][j].loss-curr_loss[prev]) /temperature));
float r = (random()) / (pow(2.0,31.0)-1.0); // why all the 2^31's ???
float ALOT=100000000000.0; // 100,000,000,000. = 10^11
- if (mlres[i][j].loss < ALOT) { // does this ever occur? if so did we want to still do the ratio computation above???
- trace("filestr.open (best_opt_some.txt, ofstream::app);");
+ if (mlres[i][j].loss < ALOT) {
+ tracef("multi_annealing: Writing: filestr.open (best_opt_some.txt, ofstream::app);\n");
# un[0]->get_target() Res.loss un[0]->get_parameter[0:4] Res.error
# filestr.open ("max_dist.txt", ofstream::app);
}
- else { tracef("Loss %f > ALOT at [i][j] = [%d][%d]\n", mlres[i][j].loss, i ,j); }
- if (r > ratio) { // Reject
+ else { // does this ever occur? if so did we want to still do the ratio computation above???
+ tracef("multi_annealing: Loss %f > ALOT at [i][j] = [%d][%d]\n", mlres[i][j].loss, i ,j);
+ }
+ if (r > ratio) { // Reject new parameter
x[i][j] = x[i-1][j];
- rejection[i][j] = rejection[i][j] + 1.0; // Is this correct? incr rejection?
+ rejection[i][j] = rejection[i-1][j] + 1.0; // Is this correct? incr rejection?
curr_loss[curr] = curr_loss[prev];
curr_sdev[curr] = curr_sdev[prev];
}
- else { // Accept
-trace("Accepting try_x[j],j=",j);
- x[i][j] = try_x[j]; # FIXME: is this right? Just assign the j'th try_x, or the whole try_x param set?
-trace("Accepting try_x[j],x[i][j]=",x[i][j]);
+ else { // Accept new parameter
+tracef("multi_annealing: Accepting try_x[j], i=%i j=%i\n",i,j);
+ x[i][j] = try_x[j];
+tracef("multi_annealing: Accepting try_x[j], i=%i j=%i try_x[j]=%f\n",i,j,try_x[j]);
curr_loss[curr] = mlres[i][j].loss;
curr_sdev[curr] = mlres[i][j].sdev;
}
@@ -144,9 +161,16 @@
}
}
-(Res r) multi_loss( float x[], float target_innov, int evolve_reruns )
+(Res r) multi_loss( int ci, int cj, float x[], float target_innov, int evolve_reruns )
{
file rfile[];
+ tracef("In multi_loss: entered: ci=%i cj=%i target_innov=%f evolve_reruns=%i x=%q\n",ci, cj, target_innov,evolve_reruns,x);
+ tracef("In multi_loss: entered: ci=%i cj=%i target_innov=%f evolve_reruns=%i\n",ci, cj, target_innov,evolve_reruns);
+/* This hangs:
+ foreach f,i in x {
+ tracef("In multi_loss: x[]: ci=%i cj=%i x[%i]=%f\n",ci, cj, i, f);
+ }
+*/
foreach i in [1:evolve_reruns] { // repeats of the evolove() - same as n_reruns
file outfile; // FIXME: map and save in future
string args[] = [ // FIXME: move this to a setargs() function
@@ -155,7 +179,7 @@
// n_epochs n_steps evolve_reruns range
// "40000", "20", @strcat(evolve_reruns), "2",
- "40000", "20", "1", " 2", # Set reruns to 1 for the cpp code; we do the re-runs in Swift for now.
+ "40000", "20", "1", "2", # Set reruns to 1 for the cpp code; we do the re-runs in Swift for now.
// verbose_level
"1",
@@ -171,23 +195,26 @@
file graph <"movie_graph.txt">;
(outfile, rfile[i]) = evolve(args,graph);
+ tracef("in multi_loss: after evolve: i=%i %k %k\n", i, outfile, rfile[i]);
}
file sumfile = sumloss(rfile);
r = readData(sumfile);
-
+ tracef("in multi_loss: returning: ci=%i cj=%i r.loss=%f r.sdev=%f\n",ci,cj,r.loss,r.sdev);
}
optimizer_sweep() // Implements logic of python driver script
{
int minrange=58;
- int maxrange=59;
+ int maxrange=209;
+ //int maxrange=59;
//int maxrange=1009;
//int maxrange=209;
int rangeinc=50;
// fixme: add provision for random priming and random param values when x[i] == -100 (see optimizer.cpp main())
- int nreps=1; # 15
+ //int nreps=1; # 15
+ int nreps=3; # 15
// file bestfile ;
// file maxfile ;
@@ -204,8 +231,8 @@
evolve_reruns = 2,
starting_jump = 2.3,
params0[] = [0.0, 0.0, 4.0, 50.0, -1.0],
- @tofloat(@strcat(target_innov)), // fixme: would @tofloat(int) work?
- annealing_cycles = 1); // fixme: clarify: annealing repeats, steps, and cycles
+ @tofloat(target_innov),
+ annealing_cycles = 1);
*/
(outfile,lossfile) = multi_annealing(
2.0,
@@ -214,8 +241,8 @@
2,
2.3,
[0.0, 0.0, 4.0, 50.0, -1.0],
- @tofloat(@strcat(target_innov)), // fixme: would @tofloat(int) work?
- 1); // fixme: clarify: annealing repeats, steps, and cycles
+ @tofloat(target_innov),
+ 1);
}
}
}
@@ -236,7 +263,7 @@
multi_annealing()
multi_loss()
evolve()
- sumloass()
+ sumloss()
Example parameter sets:
Modified: SwiftApps/SciColSim/math.swift
===================================================================
--- SwiftApps/SciColSim/math.swift 2012-01-22 03:29:59 UTC (rev 5506)
+++ SwiftApps/SciColSim/math.swift 2012-01-23 01:50:27 UTC (rev 5507)
@@ -38,7 +38,7 @@
else {
result = b;
}
- tracef("min: result=%f\n",result);
+ tracef("math/min: result=%f\n",result);
}
(float result) pow (float x, float y)
Modified: SwiftApps/SciColSim/tc
===================================================================
--- SwiftApps/SciColSim/tc 2012-01-22 03:29:59 UTC (rev 5506)
+++ SwiftApps/SciColSim/tc 2012-01-23 01:50:27 UTC (rev 5507)
@@ -1,13 +1,15 @@
localhost sh /bin/sh null null null
localhost cat /bin/cat null null null
+
pbs cat /bin/cat null null null
mcs cat /bin/cat null null null
localhost catnap /home/wilde/swift/lab/catnap.sh null null GLOBUS::maxwalltime="00:01:00"
+
beagle optimizer /home/wilde/AndreysOptimizer/Optimizer null null GLOBUS::maxwalltime="01:00:00"
beagle optimizersh /home/wilde/AndreysOptimizer/optimizer.sh null null GLOBUS::maxwalltime="02:00:00"
beagle evolve /home/wilde/AndreysOptimizer/evolve.sh null null GLOBUS::maxwalltime="02:00:00"
-localhost evolve /home/wilde/AndreysOptimizer/evolve.sh null null GLOBUS::maxwalltime="02:00:00"
+localhost evolve /home/wilde/AndreysOptimizer/src/evolve.sh null null GLOBUS::maxwalltime="02:00:00"
beagle sumloss /home/wilde/AndreysOptimizer/evolve.sh null null GLOBUS::maxwalltime="02:00:00"
-localhost sumloss /home/wilde/AndreysOptimizer/sumloss.sh null null GLOBUS::maxwalltime="02:00:00"
+localhost sumloss /home/wilde/AndreysOptimizer/src/sumloss.sh null null GLOBUS::maxwalltime="02:00:00"
From wilde at ci.uchicago.edu Mon Jan 23 12:18:10 2012
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Mon, 23 Jan 2012 12:18:10 -0600 (CST)
Subject: [Swift-commit] r5509 - SwiftApps/SciColSim
Message-ID: <20120123181810.715819CC94@svn.ci.uchicago.edu>
Author: wilde
Date: 2012-01-23 12:18:10 -0600 (Mon, 23 Jan 2012)
New Revision: 5509
Modified:
SwiftApps/SciColSim/annealing.swift
Log:
Add support for color text in messages.
Modified: SwiftApps/SciColSim/annealing.swift
===================================================================
--- SwiftApps/SciColSim/annealing.swift 2012-01-23 17:06:58 UTC (rev 5508)
+++ SwiftApps/SciColSim/annealing.swift 2012-01-23 18:18:10 UTC (rev 5509)
@@ -1,4 +1,5 @@
import "math";
+import "colortext";
type file;
@@ -121,7 +122,7 @@
tracef("in multi_annealing: i=%i j=%i k=%i\n", i, j, k);
}
// Up to here, x[] and dx[] are only set for previous i
-tracef("in multi_annealing: i=%i j=%i calling multi_loss\n",i,j);
+tracef( @strcat(color(3,"in multi_annealing: "),"i=%i j=%i calling multi_loss\n"),i,j);
#tracef("in multi_annealing: i=%i j=%i calling multi_loss try_x=%q\n",i,j,try_x);
mlres[i][j] = multi_loss(i,j,try_x, target_innov, evolve_reruns); # do the N evolve()'s, N=evolve_reruns
tracef("in multi_annealing: i=%i j=%i ret vals from multi_loss=(%f,%f)\n",i,j,mlres[i][j].loss,mlres[i][j].sdev);
From wilde at ci.uchicago.edu Mon Jan 23 12:19:35 2012
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Mon, 23 Jan 2012 12:19:35 -0600 (CST)
Subject: [Swift-commit] r5510 - SwiftApps/SciColSim
Message-ID: <20120123181935.3F8819CC94@svn.ci.uchicago.edu>
Author: wilde
Date: 2012-01-23 12:19:35 -0600 (Mon, 23 Jan 2012)
New Revision: 5510
Added:
SwiftApps/SciColSim/EMAIL
Modified:
SwiftApps/SciColSim/README
Log:
Add notes from email discussions.
Added: SwiftApps/SciColSim/EMAIL
===================================================================
--- SwiftApps/SciColSim/EMAIL (rev 0)
+++ SwiftApps/SciColSim/EMAIL 2012-01-23 18:19:35 UTC (rev 5510)
@@ -0,0 +1,474 @@
+==== Email trail (cronological order):
+
+---------------------------------------------------------------------------------------
+
+On 10/15/11 8:30 PM, "Michael Wilde" wrote:
+
+> Hi Andrey,
+>
+> I've got a basic serial version of optimizer.cpp running on Beagle (on a login
+> node). Now Im trying to parallelize it there, and have some questions:
+>
+> 1) You set NWorkers to a constant, 24. In multi_loss, you have this code:
+>
+> for(int i=0; i for(int j=0; j<5; j++){
+> un[i]->set_parameter(params[j],j);
+> }
+> for(int i=0; i dispatch_group_async(group, CustomQueues[i], ^{
+> un[i]->evolve_to_target_and_save(istart, iend, Results,
+> Counters);
+> });
+> istart += step;
+> iend = min(istart+step,N);
+> }
+> }
+>
+> Can you explain the intention here? I think the innermost loop is clear: run
+> evolve() 24 times in parallel, partitioning the istart..iend range among the
+> 24 workers. But I dont understand the outermost loop, which seems to do the
+> entire inner loop 24 (NWorkers) times. I can understand the idea of doing the
+> entire inner loop some number of times. But from the above, I presume that
+> evolve would be run NWorkers^2 times, or 24*24 times. Was that the intention?
+>
+> 2) If you had many processors available (as you do on Beagle) would you want
+> to run set NWorkers higher? I think you mentioned something in out discussion
+> Friday about "a thousand" workers being useful. Ie, NWorkers for the innermost
+> loop could be 1000. Was that really what you meant? What would be a
+> mathematically/scientifically useful value for the NWorkers in the innermost
+> loop?
+>
+> Further: if you do want an NWorkers >> 24, would you still do the evolve
+> NWorkers^2 times? I dont think you'd really want 1000*1000 = 1M evolve calls,
+> given how many outer loops you have at higher levels of the code, including in
+> the Python wrapper.
+>
+> 3) We'll need to discuss what values make sense for the outer loops
+> (multi_annealing and the .py wrapper) once I have this working in parallel.
+>
+> 4) Can you give me a sentence or two about what this code is doing? I think I
+> understand the outer annealing logic, but I have not dug into the code within
+> evolve...() at all. I think you might have explained it once back in January
+> (before you recoded in C) but I dont recall. If you have a paper or a web page
+> on what youre computing here, that would be interesting for me to read, and to
+> help write a slide on this for the Beagle review.
+
+
+-----
+
+
+Re: Status and questions on optimizer code
+From : andrey rzhetsky
+Subject : Re: Status and questions on optimizer code
+To : Michael Wilde
+Sun, Oct 16, 2011 08:28 AM
+Hi Mike,
+
+1. I think, you just uncovered a stupid bug on my part -- thank you! There
+should be only one loop (the outer one).
+2. Yes, of course -- 1000, or even 10000 (I can increase the number of
+repeats then).
+3. OK
+4. The code simulates exploration of a chemical network by a research
+community. The five major parameters determine the strategy of exploration,
+"target" is the number of new interactions discovered and the loss is the
+number of experiments per one new positive finding. I can provide you with
+figures and slides, if that would help.
+
+Thank you!
+
+With kind regards,
+
+Andrey
+
+
+---------------------------------------------------------------------------------------
+
+
+On 10/16/11 9:13 AM, "Michael Wilde" wrote:
+
+> Hi Andrey,
+>
+> Looking deeper, I think the bug was simply that the first for() statement
+> should have enclosed just the parameter setting loop. In other words, the top
+> of multi_loss should start with:
+>
+> for(int i=0; i for(int j=0; j<5; j++){
+> un[i]->set_parameter(params[j],j);
+> }
+> }
+>
+> Then the real work is done by the next loop:
+> for(i=0; i
+> Can you confirm that this looks correct to you?
+>
+> I made that change, and the code now seems to run as I would expect. I will
+> send you some output as soon as I clean up my debugging output.
+>
+> Next, I made the code run with 24-way parallelism on a Beagle login node using
+> "OpenMP", simply by adding one "pragma" statement in front of the main worker
+> loop above. So that part of the code now looks like this:
+>
+> int i;
+> #pragma omp parallel for private (i)
+> for(i=0; i
+> and each call to evolve...() is now done in parallel (with all the Mac
+> dispatch statements commented out). I will test, but I *think* that the same
+> code will run just as well in parallel on your multicore Macs, perhaps just a
+> *tiny* bit slower than under Grand Central Dispatch (likely not a noticeable
+> difference).
+>
+> Now, we have 2 choices:
+>
+> 1) I can simply replace the Python driver script with a Swift script, to do
+> many runs of the optimizer in parallel. That would give you the ability to
+> run *many* 24-core optimization runs in parallel, each using 24 cores. So for
+> example, in your current Python script you do this:
+>
+> for target in range(58,1009,50):
+> for i in range(15):
+>
+> So thats about 20 x 15 = 300 invocations of optimizer. I *think* that each of
+> these runs is totally independent and can run in parallel, correct?
+>
+> So A simple Swift script not much longer than the Python script, along with a
+> few beagle-specific configuration files, will enable all 300 jobs to run in
+> parallel, giving you 300 x 24 (=7200) cores running in parallel. Of course,
+> you can seldom *get* that many cores because the machine is heavily loaded.
+> But you may be able to get 10-30 nodes on a daily basis. We'll need to
+> experiment with this.
+>
+> As a *next* step after, we should consider the benefits of changing the value
+> of NWorkers to > 24. 24 is the "easy" limit on Beagle because we can get
+> 24-way parallelism on each node with just that one "pragma" statement. We can
+> get much greater parallelism with Swift in the inner loop, but for that we
+> need to break up the program a bit more, to have Swift run the inner loop as a
+> separate program, and then return the aggregated results in a file. Even for
+> this option, there are two alternative methods:
+>
+> - we make optimizer call Swift once for each round of parallel annealing. This
+> is fairly easy. It is somewhat limiting to overall parallelism, in that only
+> one round at a time can run. But It may be very adequate.
+>
+> - we break the program up further into parallelizable chunks, in which case
+> you have a lot of flexibility and the work always gets done in a near-optimal
+> manner regardless of the shape of a given optimization run (in terms of the
+> various nested loop sizes and evolve() execution times.
+>
+> I think we'll need to discuss this in person over a whiteboard, but I think I
+> have enough knowledge of the program to at least show you a few alternatives.
+>
+> The main question at the moment, I think, is simply to understand the
+> math/science benefits of extending NWorkers beyond the "low hanging fruit"
+> limit of 24. What is your assessment of that benefit, Andrey?
+>
+
+-----
+
+----- Forwarded Message -----
+From: "andrey rzhetsky"
+To: "Michael Wilde"
+Sent: Sunday, October 16, 2011 12:08:25 PM
+Subject: Re: Status and questions on optimizer code
+
+Mike,
+
+It would be fantastic to have 1000 or 10000 workers (with larger number of
+re-runs -- it would improve precision of my analysis drastically!).
+
+All the very best,
+
+Andrey
+
+---------------------------------------------------------------------------------------
+
+On 10/17/11 8:29 AM, "Michael Wilde" wrote:
+
+> Hi Andrey,
+>
+> Can we meet today to discuss the optimizer? I'd like to show you what Ive done
+> and discuss with you next steps towards getting you running on Beagle. I can
+> meet any time from 10:30 to 3:00.
+>
+> Do you already have a CI and Beagle login and a project set up for Beagle use?
+> If not, we should get that started.
+>
+> On the technical side, I have a question about the the typical shape of your
+> optimization runs.
+>
+> With the sample Python script you gave me, I think we have the following
+> nested iterations in the code:
+>
+> 20 targets (parallel)
+> 15 repeats (parallel)
+> 100 Annealing_cycles (serial)
+> 6 repeats (serial)
+> 1000 to 10000 annealing_repeats (parallel)
+> evolve()
+>
+> The main question I have at this point is regarding the strategy for
+> increasing the innermost annealing repeats (currently 1,000 divided among 24
+> workers; desired to increase to 10,000).
+>
+> The outermost loops in my Swift tests are done in parallel. Thus we can have a
+> 300 optimizations going in parallel and 24 annealings in parallel for a total
+> of 7,200 parallel tasks.
+>
+> The question is: if you will always have a sizeable number of parallel
+> iterations in the outer loops, we dont need to change anything in the inner
+> loop to get more parallelism. In other words, we already have more parallelism
+> than we have CPUs available.
+>
+> 7200 CPUs is about 42% of the overall Beagle system. It will be very rare
+> that we we could get that many cores all at once. But think we can regularly
+> get say 500 to 2000 cores on a daily basis.
+>
+> On the other hand, if you expect to regularly run tests of *single* annealing
+> cycles and want to speed those up, then indeed it may be worth changing the
+> code structure.
+>
+> When me meet I'll try to give you an idea of whats involved. Basically we need
+> to change the structure of the annealing loop to create a function
+> "multi_loss_setup" as a separate executable which defines the annealing
+> parameters and writes them to a file; make multi_loss a separate executable;
+> create another executable "multi_loss_summarize" which reduces the results.
+> We can probably combine multi_loss_summarize into multi_loss_setup.
+>
+> This is not very hard to do, but still sounds to me like a week of programming
+> to get all all restructured and tested. Before investing that effort, we
+> should discuss if it will give you any additional performance gains over just
+> running many optimizations in parallel.
+>
+> I need to run timings on the annealing cycles to see how that change across
+> the parameter space, to see if we can just increase the repeats to 10,000 with
+> no changes to the code. I think the feasibility of doing this the "easy way"
+> is based on how long the longest annealings take at the high end of the
+> parameter space.
+>
+> Regards,
+>
+> - Mike
+
+-----
+
+----- Forwarded Message -----
+From: "Andrey Rzhetsky"
+To: "Michael Wilde"
+Sent: Monday, October 17, 2011 8:40:17 AM
+Subject: Re: Meet today to discuss optimizer?
+
+Hi Mike,
+
+The 6 (or more) annealing repeats can be run in parallel too.
+
+Unfortunately, around 10:15 I have to rush to Evanston to CBC meeting for
+the rest of the day (we can chat before, if you have a minute, I am in my
+office).
+
+I don't have Beagle login, unfortunately.
+
+Typically, I will have a sizeable outer loop, so, probably, the current
+24-worker setup is fine.
+
+Thank you very much for helping me out!
+
+All the best,
+
+Andrey
+
+
+---------------------------------------------------------------------------------------
+
+On 10/18/11 1:52 PM, "Michael Wilde" wrote:
+
+> Hi Andrey,
+>
+> Here's a quick update:
+>
+> - I am now running the optimizer on Beagle compute nodes under Swift.
+>
+> I attach a few tar files of sample runs at reduced parameter values (to shrink
+> the run time for debugging and learning the code's behavior);
+>
+> Now Im trying to run some subset of the full-length parameters you gave me in
+> the python file. Ive got 3 Beagle compute nodes allocated at the moment (72
+> cores total) and Im seeing these times from multi_loss with N=1000 repeats:
+>
+> sandbox$ grep multi_ ./jobs/*/*/output/*.out
+> ./jobs/0/optimizer-01p7lhhk/output/T408.R1.out:multi_loss(N=1000) elapsed
+> time: 122.742 seconds 2.04571 minutes
+> ./jobs/0/optimizer-01p7lhhk/output/T408.R1.out:multi_loss(N=1000) elapsed
+> time: 123.979 seconds 2.06631 minutes
+> ./jobs/0/optimizer-01p7lhhk/output/T408.R1.out:multi_loss(N=1000) elapsed
+> time: 123.624 seconds 2.0604 minutes
+> ./jobs/t/optimizer-t0p7lhhk/output/T958.R1.out:multi_loss(N=1000) elapsed
+> time: 1431.09 seconds 23.8514 minutes
+> ./jobs/x/optimizer-x0p7lhhk/output/T708.R1.out:multi_loss(N=1000) elapsed
+> time: 627.074 seconds 10.4512 minutes
+> ./jobs/x/optimizer-x0p7lhhk/output/T708.R1.out:multi_loss(N=1000) elapsed
+> time: 790.652 seconds 13.1775 minutes
+>
+>
+> Each run of optimizer is going to a file name T(target).R(repeat).out
+>
+> So we're seeing 23.8 mins for 1000 repeats at target=958 and 10-13 mins at
+> target=708. The 1000 repeats are spread over 24 cores each.
+>
+> Whats your time availability later in the week to discuss this further, and to
+> see if either (a) I can show you how to run this version or (b) we can get a
+> set of production run descriptions from you and you can run them yourself?
+>
+> In the compressed tar file at http://www.ci.uchicago.edu/~wilde/AR.snap.01.tgz
+> you will find:
+>
+> - the swift script that I use instead of the Python driver to run the
+> optimizer in parallel (along beagle.xml that specifies scheduler parameters
+> for Beagle like time, cores, queue name and project ID)
+>
+> - the slightly modified version of optimizer (changes in multi_loss() to
+> correct the loops, changes to use OpenMP instead of Grand Central Dispatch,
+> and a few changes in output logging).
+>
+> - a few run directories of runs with shortened parameter settings.
+>
+> If we continue working together on this, we should set up a way to share code
+> using a repository like Subversion (svn). Thats pretty easy once you master a
+> few basic commands.
+>
+> Regards,
+>
+> - Mike
+>
+>
+
+-----
+
+----- Forwarded Message -----
+From: "Andrey Rzhetsky"
+To: "Michael Wilde"
+Sent: Tuesday, October 18, 2011 3:57:36 PM
+Subject: Re: Meet today to discuss optimizer?
+
+Mike,
+
+Thank you! Are you around now? I would be also happy to carve some time
+tomorrow, if this works for you.
+
+With kind regards,
+
+Andrey
+
+
+---------------------------------------------------------------------------------------
+
+
+On 10/19/11 12:10 PM, "Michael Wilde" wrote:
+
+> Hi Andrey,
+>
+> Im in meetings today till about 3PM. Are you available at say 3:30 or later?
+>
+> I did a larger run last night. Only one smaller optimizer run *fully*
+> finished, but many others made significant progress. The results are at:
+>
+> http://www.ci.uchicago.edu/~wilde/AR.optimizer.out.2010.1018.tgz
+>
+> If you have time, could you take a look at that run and see if the
+> optimizations look like they have been running as expected? Ive made only a
+> few cosmetic changes to your debug output.
+>
+> I submitted the run at 21:20; it started running at about 21:27; by about
+> 23:10 it had acquired 12 nodes (x 24 cores each). It ended about 23:18 when
+> the first job exceeded its time limit of 5 hours. Im still trying to calibrate
+> how much time each optimizer invocation needs, and whether some of the
+> internal iterations can be further spread out. Also how to organize the run
+> so that optimizations that time out can be re-run with the smallest
+> reasonable failure unit.
+>
+>
+
+---------------------------------------------------------------------------------------
+
+
+----- Forwarded Message -----
+From: "andrey rzhetsky"
+To: "Michael Wilde"
+Sent: Wednesday, October 26, 2011 8:40:21 PM
+Subject: Re: Question on inner annealing loop
+
+Mike,
+
+
+> Im confused on 3 points here:
+>
+> - the inner loop would always be done between 1 and 5 times, right?
+
+Correct.
+
+> - could each of those times really be done in parallel? (I'll try to determine
+> this by inspection).
+
+Not really -- the acceptance of parameter changes depends on the loss in
+between.
+
+> - when we last met in your office, I *thought* you indicated that this inner
+> loop could be done just *once*. Was that what you meant? And if so, for
+> which of the 5 vars?
+
+Nope, has to be repeated over and over.
+
+All the very best,
+
+Andrey
+
+
+---------------------------------------------------------------------------------------
+
+
+On 10/26/11 10:42 PM, "Michael Wilde" wrote:
+
+> OK, all that makes sense, Andrey. But then do you recall what you suggested
+> when we met?
+>
+> Lets label the loops as follows:
+>
+> a) 20 targets (parallel)
+> b) 15 repeats (parallel)
+> c) 100 Annealing_cycles (serial)
+> d) 6 repeats (serial)
+> e) 1000 to 10000 annealing_repeats (parallel)
+> f) evolve()
+>
+> What I recalled from our last discussion was that I should reduce loop (c)
+> from 100 to 50 or 25, and loop (d) to 1. But since reducing loop (d) doesn't
+> make sense, do you recall suggesting any other reduction?
+>
+> If not, no problem, I think I know how to proceed.
+>
+> Thanks,
+>
+> - Mike
+>
+>
+-----
+
+----- Forwarded Message -----
+From: "andrey rzhetsky"
+To: "Michael Wilde"
+Sent: Thursday, October 27, 2011 2:54:06 AM
+Subject: Re: Question on inner annealing loop
+
+Hi Mike,
+
+I suggested reducing (b) to 1.
+
+With kind regards,
+
+Andrey
+
+
+
Modified: SwiftApps/SciColSim/README
===================================================================
--- SwiftApps/SciColSim/README 2012-01-23 18:18:10 UTC (rev 5509)
+++ SwiftApps/SciColSim/README 2012-01-23 18:19:35 UTC (rev 5510)
@@ -1,13 +1,34 @@
+=== Overview
+
+The code simulates the exploration of a chemical network by a research
+community. The five major parameters determine the strategy of
+exploration, "target" is the number of new interactions discovered and
+"loss" is the number of experiments per one new positive finding.
+
=== Files
+Code:
+
optirun.swift: replaces top-level py script for the outermost loops
+Data:
+
+
=== How to build
+make
+
=== How to Run
+./test0.sh
+./testO.sh
+./test1.sh
+
+./*.py
+
+
=== C++ app flow logic ===
for target in range(58, 1009 (used 209), 50): // 20 values
@@ -36,7 +57,9 @@
1 initial multi_loss: 1000 to 10000 annealing_repeats
100 Annealing_cycles (groups of 10? : cycle=10 ) (fast:50)
5 repeats (fast: 1)
- multi_loss: 1000 to 10000 annealing_repeats
+ multi_loss: 1000 to 10000 evolve re-runs
evolve() => 2 mins to 10 mins
=== END
+
+
From wilde at ci.uchicago.edu Mon Jan 23 12:30:50 2012
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Mon, 23 Jan 2012 12:30:50 -0600 (CST)
Subject: [Swift-commit] r5512 - SwiftApps/SciColSim
Message-ID: <20120123183050.4EAD19CC94@svn.ci.uchicago.edu>
Author: wilde
Date: 2012-01-23 12:30:50 -0600 (Mon, 23 Jan 2012)
New Revision: 5512
Added:
SwiftApps/SciColSim/optimizer.cpp
Log:
Original version from Andrey: base for re-applying necessary revisions from prototype to create new production code.
Added: SwiftApps/SciColSim/optimizer.cpp
===================================================================
--- SwiftApps/SciColSim/optimizer.cpp (rev 0)
+++ SwiftApps/SciColSim/optimizer.cpp 2012-01-23 18:30:50 UTC (rev 5512)
@@ -0,0 +1,1674 @@
+//
+// main.cpp
+// optimizer
+//
+// Created by Andrey Rzhetsky on 4/11/11.
+// Copyright 2011 University of Chicago. All rights reserved.
+//
+
+#define Nworkers 24
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include
+#include
+#include
+#include
+
+#include
+#include
+
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include
+#include
+#include
+
+#define BOOST_MATH_OVERFLOW_ERROR_POLICY ignore_error
+#define BOOST_MATH_DISCRETE_QUANTILE_POLICY real
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#define INT_INFINITY 2147483647
+
+#define FIX_VARIABLES 1
+
+using namespace boost;
+using namespace std;
+using namespace boost::numeric::ublas;
+
+static int max_dist=0;
+
+typedef boost::adjacency_matrix Graph;
+typedef std::pair Edge;
+typedef boost::graph_traits GraphTraits;
+typedef boost::numeric::ublas::triangular_matrix prob;
+typedef boost::numeric::ublas::triangular_matrix pathlength;
+typedef boost::graph_traits::vertex_descriptor vertex_descriptor;
+
+namespace std {
+ using ::time;
+}
+
+static int var_fixed[5] = {1, 0, 1, 1, 0};
+
+typedef boost::minstd_rand base_generator_type;
+typedef adjacency_list < listS, vecS, directedS,
+no_property, property < edge_weight_t, int > > graph_t;
+typedef graph_traits < graph_t >::vertex_descriptor vertex_descriptor;
+typedef graph_traits < graph_t >::edge_descriptor edge_descriptor;
+
+
+//================================================
+string strDouble(double number)
+{
+ stringstream ss;//create a stringstream
+ ss << number;//add number to the stream
+ return ss.str();//return a string with the contents of the stream
+}
+
+//================================================
+
+double gaussian(double sigma)
+{
+ double GaussNum = 0.0;
+ int NumInSum = 10;
+ for(int i = 0; i < NumInSum; i++)
+ {
+ GaussNum += ((double)rand()/(double)RAND_MAX - 0.5);
+ }
+ GaussNum = GaussNum*sqrt((double)12/(double)NumInSum);
+
+
+ return GaussNum;
+
+}
+
+
+
+//=================================================
+double diffclock(clock_t clock1,clock_t clock2)
+{
+ double diffticks=clock1-clock2;
+ double diffms=(diffticks)/CLOCKS_PER_SEC;
+ return diffms;
+}
+
+//================================================
+//================================================================
+double get_new_x(double x, double dx){
+
+ double new_x;
+ // boost::variate_generator > uni(generator, uni_dist);
+ double r = rand()/(double)(pow(2.,31)-1.);
+
+ if (r > 0.5){
+ new_x = x + rand()*dx/(double)(pow(2.,31)-1.);
+ } else {
+ new_x = x - rand()*dx/(double)(pow(2.,31)-1.);
+ }
+
+ return new_x;
+
+}
+
+
+//===============================================
+string string_wrap(string ins, int mode){
+
+ std::ostringstream s;
+
+ switch(mode){
+ case 0:
+ s << "\033[1;29m" << ins << "\033[0m";
+ break;
+ case 1:
+ s << "\033[1;34m" << ins << "\033[0m";
+ break;
+ case 2:
+ s << "\033[1;44m" << ins << "\033[0m";
+ break;
+ case 3:
+ s << "\033[1;35m" << ins << "\033[0m";
+ break;
+ case 4:
+ s << "\033[1;33;44m" << ins << "\033[0m";
+ break;
+ case 5:
+ s << "\033[1;47;34m" << ins << "\033[0m";
+ break;
+ case 6:
+ s << "\033[1;1;31m" << ins << "\033[0m";
+ break;
+ case 7:
+ s << "\033[1;1;33m" << ins << "\033[0m";
+ break;
+ case 8:
+ s << "\033[1;1;43;34m" << ins << "\033[0m";
+ break;
+ case 9:
+ s << "\033[1;1;37m" << ins << "\033[0m";
+ break;
+ case 10:
+ s << "\033[1;30;47m" << ins << "\033[0m";
+ break;
+ default:
+ s << ins;
+ }
+
+ return s.str();
+}
+
+
+//===============================================
+string wrap_double(double val, int mode){
+
+ std::ostringstream s;
+ s << string_wrap(strDouble(val),mode);
+
+ return s.str();
+}
+
+
+
+//===============================================
+const
+string i2string(int i){
+
+ std::ostringstream s;
+ s << "worker"
+ << lexical_cast(i);
+
+ return s.str();
+
+}
+
+//===============================================
+char* i2char(int i){
+
+ std::ostringstream s;
+ s << "worker"
+ << lexical_cast(i);
+
+ char* a=new char[s.str().size()+1];
+ memcpy(a,s.str().c_str(), s.str().size());
+
+ return a;
+}
+
+//================================================
+class Universe {
+
+private:
+
+ double alpha_i;
+ double alpha_m;
+ double beta;
+ double gamma;
+ double delta;
+
+ double TargetNovelty;
+ double CumulativeRelativeLoss;
+ double CRLsquare;
+ string id;
+
+
+ int N_nodes;
+ int M_edges;
+
+ int N_epochs;
+ int N_steps;
+ int N_repeats;
+
+ int current_epoch;
+ double current_loss;
+ int current_repeat;
+ double current_novelty;
+
+ int mode_identify_failed;
+ int verbose_level; // 0 is silent, higher is more
+
+ double k_max;
+
+ graph_t Full_g;
+
+ double **Prob;
+ double **Tried;
+ double **Dist;
+ double **Final;
+ double **EdgeIndex;
+ double *Rank;
+
+ base_generator_type generator;
+ boost::uniform_real<> uni_dist;
+ boost::geometric_distribution geo;
+
+public:
+
+
+
+ //====== Constructor ======
+ Universe(const std::string FileToOpen, int Epochs, int Steps, int Repeats, int identify_failed, double target, const std::string idd)
+ {
+ //typedef array_type2::index index2;
+
+
+ std::ifstream inFile;
+ //string line;
+
+ //-------------------------------
+
+ base_generator_type gene(42u);
+ generator = gene;
+ generator.seed(static_cast(std::time(0)));
+ boost::uniform_real<> uni_d(0,1);
+ uni_dist = uni_d;
+
+ //--------------------------------
+
+ int i, k;
+ int x, y;
+ Edge* edge_array_mine;
+ int num_arcs_mine, num_nodes_mine;
+ int* weights_mine;
+
+ TargetNovelty = target;
+ CumulativeRelativeLoss = 0.;
+ CRLsquare = 0.;
+
+
+ N_epochs = Epochs;
+ N_steps = Steps;
+ N_repeats = Repeats;
+
+ current_epoch = 0;
+ current_loss = 0.;
+ current_repeat = 0;
+
+ id = idd;
+
+ verbose_level = 1;
+
+ mode_identify_failed = identify_failed;
+
+
+ //-------------------------------
+ // The first pass though file with the graph
+ inFile.open(FileToOpen.c_str());
+ if (inFile.fail()) {
+ cout << "Unable to open file";
+ exit(1); // terminate with error
+ }else {
+
+ if (verbose_level > 2){
+ std::cout << " Opened <" << FileToOpen << ">"<> x;
+ inFile >> y;
+
+ if (verbose_level > 2){
+ std::cout << " x: " << x;
+ std::cout << " y: " << y << std::endl;
+ }
+
+ if (i==0){
+ N_nodes=x;
+ M_edges=y;
+ break;
+ }
+ i++;
+
+
+ }
+ inFile.close();
+
+ if (verbose_level == 2){
+ std::cout << N_nodes << " nodes, " << M_edges << " edges"< 2){
+ std::cout << " Opened <" << FileToOpen << ">"<> x && inFile >>y) {
+ if (i > 0) {
+ Final[x][y]=1.;
+ Final[y][x]=1.;
+
+
+ if (verbose_level == 2){
+ std::cout << ".";
+ }
+ }
+ i++;
+
+ }
+ if (verbose_level == 2){
+ std::cout << std::endl;
+ }
+ inFile.close();
+
+ k=0;
+ for (int i=0; i 0.){
+ EdgeIndex[i][j]=k;
+ k++;
+ }
+ }
+ }
+
+
+
+ //+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ // create graph -- hopefully, we can keep it, just modifying edge weights
+
+
+ edge_array_mine = new Edge[2*M_edges];
+ num_arcs_mine = 2*M_edges;
+ num_nodes_mine = N_nodes;
+ weights_mine = new int[2*M_edges];
+ for (int i=0; i<2*M_edges; i++){ weights_mine[i]=1;}
+
+ k=0;
+ for(int i=0; i0.){
+ edge_array_mine[2*k] =Edge(i,j);
+ edge_array_mine[2*k+1]=Edge(j,i);
+ k++;
+ }
+ }
+ }
+ graph_t g(edge_array_mine, edge_array_mine + num_arcs_mine, weights_mine, num_nodes_mine);
+
+ Full_g = g;
+ delete edge_array_mine;
+ delete weights_mine;
+
+ //===========================================================================
+ std::vector p(num_edges(Full_g));
+ std::vector d(num_edges(Full_g));
+ edge_descriptor s;
+ boost::graph_traits::vertex_descriptor u, v;
+
+ for (int i=0; i 0.){
+ u = vertex(i, Full_g);
+ v = vertex(j, Full_g);
+ remove_edge(u,v,Full_g);
+ remove_edge(v,u,Full_g);
+
+ }
+ }
+ }
+
+
+ }
+
+
+ //=====================================================================
+ int sample_failed_number(double pfail){
+
+ //boost::geometric_distribution geo(pfail);
+ //boost::variate_generator > geom(generator, geo);
+
+ double r, u, g;
+
+ r=0.;
+ for(int i=0; i=3){
+ std::cout << id << " failed " << r << std::endl;
+ }
+ return r;
+
+ }
+
+ //=============================================
+ double get_target(void){
+ return TargetNovelty;
+ }
+
+ //=============================================
+ void set_target(double target){
+ TargetNovelty=target;
+ }
+
+ //=============================================
+ int sample(){
+
+ //boost::variate_generator > uni(generator, uni_dist);
+ // double r = uni(), Summa = 0.;
+
+
+
+ double r = rand(), Summa = 0.;
+ r /= (double)(pow(2.,31)-1.);
+ int result = 0;
+ int finished = 0;
+
+ if (verbose_level==4){
+ std::cout << id << " sampled " << r << std::endl;
+ }
+
+ for(int i=0; i r){
+
+ Tried[i][j]+=1.;
+
+ if (Final[i][j] > 0.){
+ result = 1;
+ }
+ finished = 1;
+ }
+ }
+ }
+
+ return result;
+
+ }
+
+ //===============================
+ void update_current_graph(void){
+
+ std::vector p(num_edges(Full_g));
+ std::vector d(num_edges(Full_g));
+ edge_descriptor s;
+ boost::graph_traits::vertex_descriptor u, v;
+
+ //property_map::type weightmap = get(edge_weight, Full_g);
+ for (int i=0; i 0. && Tried[i][j]>0){
+ //s = edge(i, j, Full_g);
+ boost::graph_traits::edge_descriptor e1,e2;
+ bool found1, found2;
+ u = vertex(i, Full_g);
+ v = vertex(j, Full_g);
+ tie(e1, found1) = edge(u, v, Full_g);
+ tie(e2, found2) = edge(v, u, Full_g);
+ if (!found1 && !found2){
+ add_edge(u,v,1,Full_g);
+ add_edge(v,u,1,Full_g);
+ }
+
+ }
+ }
+
+ }
+ }
+
+ //===============================
+ void update_distances(void){
+ // put shortest paths to the *Dist[][]
+ std::vector p(num_vertices(Full_g));
+ std::vector d(num_vertices(Full_g));
+ vertex_descriptor s;
+
+
+ // put shortest paths to the *Dist[][]
+ for (int j=0; j 0.){
+ s = vertex(j, Full_g);
+ dijkstra_shortest_paths(Full_g, s, predecessor_map(&p[0]).distance_map(&d[0]));
+
+ //std::cout <<" Vertex "<< j << std::endl;
+ graph_traits < graph_t >::vertex_iterator vi, vend;
+
+ for (boost::tie(vi, vend) = vertices(Full_g); vi != vend; ++vi) {
+
+ if (p[*vi]!=*vi){
+ Dist[*vi][j]=d[*vi];
+ Dist[j][*vi]=d[*vi];
+
+ if (Dist[*vi][j]>max_dist){
+ max_dist=Dist[*vi][j];
+ }
+
+
+ } else {
+ Dist[*vi][j]=-1.;
+ Dist[j][*vi]=-1.;
+ }
+ }
+ }
+
+ }
+
+
+ }
+
+ //======================================================
+ void update_ranks(void){
+
+ for(int i=0; i0. && Final[i][j] >0.){
+ Rank[i]++;
+ Rank[j]++;
+ }
+ }
+ }
+
+ }
+
+ //====================================================================
+ void set_world(double a_i, double a_m, double b, double g, double d){
+
+ alpha_i=a_i;
+ alpha_m=a_m;
+ gamma=g;
+ beta=b;
+ delta=d;
+
+ }
+
+ //====================================================================
+ void reset_world(){
+
+ //====================================================
+ std::vector p(num_edges(Full_g));
+ std::vector d(num_edges(Full_g));
+ edge_descriptor s;
+ boost::graph_traits::vertex_descriptor u, v;
+
+
+ for (int i=0; i 0. && Tried[i][j] > 0){
+ u = vertex(i, Full_g);
+ v = vertex(j, Full_g);
+ remove_edge(u,v,Full_g);
+ remove_edge(v,u,Full_g);
+
+ }
+ }
+ }
+
+ //==================================================
+
+ current_loss=0;
+ current_epoch=0;
+ current_repeat++;
+ current_novelty=0;
+
+ for(int i = 0; i < N_nodes; ++i) {
+ Rank[i]=0.;
+ for(int j = 0; j < N_nodes; ++j) {
+ Prob[i][j]=0.;
+ Dist[i][j]=-1.;
+ Tried[i][j]=0.;
+ }
+ }
+ }
+
+
+ //==============================================
+ void show_parameters(void){
+
+ std::cout << "Parameters: "
+ << alpha_i << " "
+ << alpha_m << " | "
+ << beta << " "
+ << gamma << " | "
+ << delta << std::endl;
+
+ }
+
+
+
+ //===============================================
+ string file_name(){
+
+ std::ostringstream s;
+ s << "world_"
+ << lexical_cast(alpha_i) << "_"
+ << lexical_cast(alpha_m) << "_"
+ << lexical_cast(beta) << "_"
+ << lexical_cast(gamma) << "_"
+ << lexical_cast(delta) << "_"
+ << lexical_cast(N_epochs) << "_"
+ << lexical_cast(N_steps) << "_"
+ << lexical_cast(N_repeats) << ".txt";
+
+ return s.str();
+
+ }
+
+
+
+
+ //=================================================
+ void set_verbose(int verbose){
+
+ verbose_level = verbose;
+ }
+
+
+ //=============================================================
+ void update_probabilities(void){
+
+
+ //=========================
+ // Compute sampling probabilities
+ // first pass: \xi_i,j
+ for(int i=0; i 0.){
+
+ double k = Dist[i][j];
+ if (k >= k_max){
+ k = k_max-1;
+ }
+
+ bg = beta * log(k/k_max) + gamma * log(1. - k/k_max);
+
+ } else {
+ bg = delta;
+ }
+
+ Prob[i][j] = exp(Prob[i][j] + bg);
+ }
+ }
+
+
+ // second pass: sum
+ double Summa = 0.;
+
+ for(int i=0; i0. && Final[i][j]>0.){
+ novel+=1.;
+ }
+ }
+ }
+
+ }
+ //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ else {
+
+ double pfail=0.;
+ int n_failed;
+ //, n_check = 0;
+
+ for(int i=0; i0. && Final[i][j]>0.){
+ novel+=1.;
+ }
+ }
+ }
+ }
+
+ current_novelty = novel;
+
+
+ //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ if (verbose_level == 2){
+ std::cout << (current_repeat+1) << " epoch=" << (current_epoch+1)
+
+ << " cost=" << cost
+ << " novel=" << novel
+ << " rel_loss=" << cost/novel
+ << std::endl;
+ }
+
+ current_epoch++;
+ }
+
+
+ //====== Destructor ======
+ ~Universe(){
+
+ delete_2Dmatrix(Final, N_nodes);
+ delete_2Dmatrix(Dist, N_nodes);
+ delete_2Dmatrix(Tried, N_nodes);
+ delete_2Dmatrix(Prob, N_nodes);
+ delete_2Dmatrix(EdgeIndex, N_nodes);
+ delete_1Dmatrix(Rank);
+ }
+
+ //================================================
+ // Allocate memory
+ double** allocate_2Dmatrix(int N, int M)
+ {
+ double **pointer;
+
+ if (verbose_level == 2){
+ std::cout<< "["< 0){
+
+ pointer = new double[N];
+
+ }else {
+
+ pointer = NULL;
+ }
+
+ return pointer;
+
+ }
+
+ //==============================================
+ // De-Allocate memory to prevent memory leak
+ void delete_2Dmatrix(double **pointer, int N){
+
+ if (pointer != NULL){
+
+ for (int i = 0; i < N; ++i){
+ delete [] pointer[i];
+ }
+ delete [] pointer;
+ }
+ }
+ //====================
+ void delete_1Dmatrix(double *pointer){
+
+ delete [] pointer;
+ }
+
+ //===========================================
+ double get_rel_loss(){
+
+ return CumulativeRelativeLoss ;
+ }
+
+ //===========================================
+ double get_rel_loss_err(){
+
+ return CRLsquare ;
+ }
+
+
+
+ //==================================================================================
+ void evolve_to_target_and_save(int istart, int iend, double* storage, int* counters){
+
+ double ALOT=100000000000.;
+
+ reset_world();
+
+ for (int k = istart; k < iend; k++){
+
+
+ for(int i=0; i< N_epochs && current_novelty < TargetNovelty; i++){
+ update_world();
+ }
+
+ storage[k]=current_loss/current_novelty;
+ counters[k]=1;
+
+
+ reset_world();
+ }
+
+ }
+ //==============================================
+ int get_reruns(void){
+ return N_repeats;
+ }
+
+ //==============================================
+ double get_parameter(int i){
+
+ switch(i){
+ case 0:
+ return alpha_i;
+ case 1:
+ return alpha_m;
+ case 2:
+ return beta;
+ case 3:
+ return gamma;
+ case 4:
+ return delta;
+ default:
+
+ std::cout << "Erroneous parameter id!!!!\n\n\n";
+ return 0.;
+ }
+ }
+
+
+ //==============================================
+ void evolve_to_target(){
+
+ reset_world();
+ if (beta < -1. || gamma < -1.){
+ CumulativeRelativeLoss = 100000000000.;
+ CRLsquare = 0.;
+ return;
+ }
+
+
+ for (int k=0; k< N_repeats; k++){
+
+
+ for(int i=0; i 4) {return 0;}
+
+ else {
+
+ switch(position){
+ case 0:
+ alpha_i=value;
+ return 1;
+ case 1:
+ alpha_m=value;
+ return 1;
+ case 2:
+ beta=value;
+ return 1;
+ case 3:
+ gamma=value;
+ return 1;
+ case 4:
+ delta=value;
+ return 1;
+ }
+
+ }
+
+ return 0;
+ }
+
+
+ //=================================================================
+ void try_annealing(double starting_jump, int iterations,
+ double temp_start, double temp_end, double target_rejection){
+
+ double dx[5]={0.,0.,0.,0.,0};
+ double x[5]={0.,0.,0.,0.,0};
+ double rejection[5]={0., 0., 0., 0., 0.};
+ double curr_x, curr_err, x_tmp;
+ double temperature;
+ double ratio, r;
+ int cycle=10;
+ boost::variate_generator > uni(generator, uni_dist);
+
+ // set up parameter for annealing
+
+ x[0]=alpha_i;
+ x[1]=alpha_m;
+ x[2]=beta;
+ x[3]=gamma;
+ x[4]=delta;
+
+ for(int i=0;i<5;i++){
+ dx[i] = starting_jump;
+ }
+
+ // establish the current value
+
+ //..........................................
+ evolve_to_target();
+ std::cout << CumulativeRelativeLoss << " +- " << CRLsquare << std::endl;
+
+ curr_x = CumulativeRelativeLoss;
+ curr_err = CRLsquare;
+ CumulativeRelativeLoss = 0;
+ CRLsquare = 0;
+ //...........................................
+
+ // optimization cycle
+ for(int i=0; i 0){
+
+ for (int k=0; k<5; k++){
+
+ rejection[k]/=(double)cycle;
+ if (rejection[k] > 0){
+ dx[k] = dx[k]/(rejection[k]/target_rejection);
+ rejection[k]=0.;
+ }
+ else{
+ dx[k]*=2.;
+ }
+ std::cout << dx[k] << " ";
+ }
+ std::cout << std::endl;
+ }
+
+
+ for (int j=0; j<5; j++){
+
+ // get new value of x[j]
+ x_tmp = get_new_x(x[j],dx[j]);
+
+
+
+ //.............................................
+ set_parameter(x_tmp, j);
+
+
+ evolve_to_target();
+
+ std::cout << std::endl << "......... " << std::endl;
+ std::cout << "Trying... " << CumulativeRelativeLoss << " +- " << CRLsquare << std::endl;
+
+ ratio = min(1.,exp(-(CumulativeRelativeLoss-curr_x)/temperature));
+ r = uni();
+ std::cout << r << " vs " << ratio << std::endl;
+
+ if (r > ratio){
+
+ std::cout << string_wrap(id, 4) <<" "<< (i+1) << ","<< (j)
+ <<" "<< (i+1) << " Did not accept "
+ << x_tmp << "(" << j << ")" << std::endl;
+ std::cout << alpha_i << " "<< alpha_m << " "
+ << beta << " " << gamma << " "
+ << delta << " " << std::endl;
+ set_parameter(x[j], j);
+ CumulativeRelativeLoss = 0;
+ CRLsquare = 0;
+
+ rejection[j]+=1.;
+ }
+
+ else {
+
+ curr_x = CumulativeRelativeLoss;
+ curr_err = CRLsquare;
+ x[j] = x_tmp;
+ CumulativeRelativeLoss = 0;
+ CRLsquare = 0;
+ std::cout << (i+1) << string_wrap((string) " Rejection counts: ", 8)
+ << wrap_double(rejection[0],2)
+ << " "<< wrap_double(rejection[1], 7) << " "
+ << wrap_double(rejection[2],5) << " " << wrap_double(rejection[2],9) << " "
+ << wrap_double(rejection[4],6) << " "
+ << std::endl << std::endl;
+
+ std::cout << string_wrap(id, 4) <<" "<< (i+1) <<","<< (j)
+ <<" "
+ << string_wrap((string) "***** Did accept! ", 3)
+ << wrap_double(alpha_i,2)
+ << " "<< wrap_double(alpha_m, 7) << " "
+ << wrap_double(beta,5) << " "
+ << wrap_double(gamma,9) << " "
+ << wrap_double(delta,6) << " "
+ << std::endl << std::endl;
+
+ }
+ //........................................................
+
+ }
+
+ }
+
+ }
+
+
+};
+
+//============================================================
+
+std::pair multi_loss(dispatch_group_t group,
+ Universe* un[],
+ dispatch_queue_t* CustomQueues,
+ double* Results,
+ int* Counters,
+ double* params){
+
+ int N = un[0]->get_reruns();
+ int step = (int)(double)N/(double)(Nworkers);
+ int istart=0;
+ int iend = istart+step;
+
+ double Loss=0., LossSquare=0.;
+
+ for(int i=0; iset_parameter(params[j],j);
+ }
+
+
+ for(int i=0; ievolve_to_target_and_save(istart, iend, Results, Counters);
+ });
+
+ istart += step;
+ iend = min(istart+step,N);
+
+ }
+ }
+ dispatch_group_wait(group, DISPATCH_TIME_FOREVER);
+ //dispatch_release(group);
+
+ for (int i=0; i Res;
+ Res.first=Loss;
+ Res.second=two_std;
+
+ return Res;
+
+
+}
+//============================================================
+
+
+//============================================================
+void multi_annealing( dispatch_group_t group,
+ Universe* un[],
+ dispatch_queue_t* CustomQueues,
+ double T_start, double T_end,
+ double Target_rejection,
+ int Annealing_repeats,
+ double starting_jump,
+ double* Results,
+ int* Counters,
+ double* params0,
+ double annealing_cycles){
+ //.................................
+ // re-implement annealing
+
+ double dx[5]={0.,0.,0.,0.,0};
+ double x[5]={0.,0.,0.,0.,0};
+ double rejection[5]={0., 0., 0., 0., 0.};
+ double curr_x, curr_err, x_tmp;
+ double temperature;
+ double ratio, r;
+ int cycle=10;
+ //boost::variate_generator > uni(generator, uni_dist);
+
+ // set up parameter for annealing
+
+ x[0]=params0[0];
+ x[1]=params0[1];
+ x[2]=params0[2];
+ x[3]=params0[3];
+ x[4]=params0[4];
+
+ for(int i=0;i<5;i++){
+ dx[i] = starting_jump;
+ }
+
+ // establish the current value
+ std::pairRes;
+
+ Res = multi_loss(group, un, CustomQueues, Results, Counters, x);
+ std::cout << Res.first << " +- " << Res.second << std::endl;
+
+ curr_x = Res.first;
+ curr_err = Res.second;
+
+ // optimization cycle
+
+ for(int i=0; i 0){
+
+ for (int k=0; k<5; k++){
+ rejection[k]/=(double)cycle;
+
+ if (rejection[k] > 0){
+ dx[k] = dx[k]/(rejection[k]/Target_rejection);
+ rejection[k]=0.;
+ }
+ else{
+ dx[k]*=2.;
+ }
+ std::cout << dx[k] << " ";
+ }
+ std::cout << std::endl;
+ }
+
+
+ for (int j=0; j<5; j++){
+
+ ///////////////////////////////
+ if (FIX_VARIABLES==0 || var_fixed[j]==0){
+
+
+
+ // get new value of x[j]
+ double x_hold=x[j];
+ x_tmp = get_new_x(x[j],dx[j]);
+ x[j]=x_tmp;
+
+ std::cout << wrap_double(x_tmp,10) << " " << wrap_double(j,9) << "\n\n";
+ //=======================================
+ //.............................................
+ for(int w=0; wset_parameter(x_tmp, j);
+ }
+
+
+ Res = multi_loss(group, un, CustomQueues, Results, Counters, x);
+ std::cout << Res.first << " +- " << Res.second << std::endl;
+
+ ratio = min(1.,exp(-(Res.first-curr_x)/temperature));
+ r = rand()/(double)(pow(2.,31)-1.);
+ std::cout << r << " vs " << ratio << std::endl;
+
+ double ALOT=100000000000.;
+
+ if (Res.first < ALOT)
+ {
+ ofstream filestr;
+
+ filestr.open ("best_opt_some.txt", ofstream::app);
+
+ // >> i/o operations here <<
+ filestr << un[0]->get_target() << ","
+ << Res.first
+ << "," << un[0]->get_parameter(0)
+ << "," << un[0]->get_parameter(1)
+ << "," << un[0]->get_parameter(2)
+ << "," << un[0]->get_parameter(3)
+ << "," << un[0]->get_parameter(4) << "," << Res.second << ",\n";
+
+ filestr.close();
+
+
+ filestr.open ("max_dist.txt", ofstream::app);
+
+ // >> i/o operations here <<
+ filestr << max_dist << ",\n";
+
+ filestr.close();
+
+ }
+
+
+ if (r > ratio){
+
+ std::cout << " "<< (i+1) << ","<< (j)
+ <<" "<< (i+1) << " Did not accept "
+ << x_tmp << "(" << j << ")" << std::endl;
+ std::cout << un[0]->get_parameter(0)
+ << " " << un[0]->get_parameter(1)
+ << " " << un[0]->get_parameter(2)
+ << " " << un[0]->get_parameter(3)
+ << " " << un[0]->get_parameter(4) << " " << std::endl;
+
+ x[j]=x_hold;
+ for(int w=0; wset_parameter(x[j], j);
+ }
+
+
+ //set_parameter(x[j], j);
+ rejection[j]+=1.;
+ }
+
+ else {
+
+ curr_x = Res.first;
+ curr_err = Res.second;
+ x[j] = x_tmp;
+
+ for(int w=0; wset_parameter(x[j], j);
+ }
+
+ std::cout << (i+1) << string_wrap((string) " Rejection counts: ", 8)
+ << wrap_double(rejection[0],2) << " "
+ << wrap_double(rejection[1],7) << " "
+ << wrap_double(rejection[2],5) << " "
+ << wrap_double(rejection[3],9) << " "
+ << wrap_double(rejection[4],6) << " "
+ << std::endl << std::endl;
+
+ std::cout << " "<< (i+1) <<","<< (j)
+ <<" "
+ << string_wrap((string) "***** Did accept! ", 3)
+ << wrap_double(un[0]->get_parameter(0),2) << " "
+ << wrap_double(un[0]->get_parameter(1),7) << " "
+ << wrap_double(un[0]->get_parameter(2),5) << " "
+ << wrap_double(un[0]->get_parameter(3),9) << " "
+ << wrap_double(un[0]->get_parameter(4),6) << " "
+ << std::endl << std::endl;
+
+
+
+ }
+ //........................................................
+
+ }
+ }
+
+ }
+
+}
+
+
+
+//================================================
+int
+main(int argc, char* argv[])
+{
+
+ double params0[6] = {0., 0., 0., 0., 0., 0.2}, target=50., range;
+ string par_names0[6] = {"alpha_i", "alpha_m", "beta", "gamma", "delta", "target"};
+ string par_names1[4] = {"n_epochs", "n_steps", "n_reruns", "range"};
+ string par_names2[5] = {"T_start", "T_end", "Annealing_steps","Target_rejection","Starting_jump"};
+ string par_names3[5] = {"FREEZE_alpha_i", "FREEZE_alpha_m", "FREEZE_beta", "FREEZE_gamma", "FREEZE_delta"};
+ int params1[4] = {300, 50, 1000, 10};
+ int params3[5] = { 0, 0, 0, 0, 0};
+
+ // temperature_start, temperature_end, annealing_steps target_rejection Starting_jump
+ double params2[5] = {1, 0.001, 100, 0.3, 1.5};
+
+ int verbose_level = 2;
+ const std::string one="one", two="two";
+ static Universe* un[Nworkers];
+ static dispatch_queue_t CustomQueues[Nworkers];
+
+ static double* Results;
+ static int* Counters;
+
+ timeval t1, t2;
+ double elapsedTime;
+ // start timer
+ gettimeofday(&t1, NULL);
+
+
+ if (argc < 8) {
+ std::cout << "Usage: super_optimizer alpha_i alpha_m beta gamma delta target_innov [n_epochs n_steps n_reruns] [range] [verbose_level]\n";
+ std::cout << " [T_start T_end Annealing_steps Target_rejection Starting_jump]\n";
+ std::cout << " [FREEZE_alpha_i FREEZE_alpha_m FREEZE_beta FREEZE_gamma FREEZE_delta]\n";
+
+ system("pwd");
+
+
+ return(1);
+ }
+ else {
+ for (int nArg=0; nArg < argc; nArg++){
+ //std::cout << nArg << " " << argv[nArg] << std::endl;
+ if (nArg > 0 && nArg < 7){
+ params0[nArg-1]= atof(argv[nArg]);
+ std::cout << par_names0[nArg-1] << ": " << params0[nArg-1] << std::endl;
+ }
+ if (nArg > 6 && nArg < 11){
+ params1[nArg-7]= atoi(argv[nArg]);
+ std::cout << par_names1[nArg-7] << ": " << params1[nArg-7] << std::endl;
+ }
+ if (nArg == 11){
+ verbose_level = atoi(argv[nArg]);
+ std::cout << "verbose level: " << verbose_level << std::endl;
+ }
+ if (nArg > 11 && nArg < 17){
+ params2[nArg-12]= atof(argv[nArg]);
+ std::cout << par_names2[nArg-12] << ": " << params2[nArg-12] << std::endl;
+ }
+ if (nArg > 16 && nArg < 22){
+ params3[nArg-17]= atof(argv[nArg]);
+ var_fixed[nArg-17]= atof(argv[nArg]);
+ std::cout << par_names3[nArg-17] << ": " << var_fixed[nArg-17] << std::endl;
+ }
+
+
+ }
+
+ }
+
+ for (int j=0; j<5; j++){
+
+ cout << j << " | " << var_fixed[j] << " (fixed) \n";
+ }
+
+ target=params0[5];
+ range = (double)params1[3];
+ int identify_failed = 0;
+ char* filename= (char *)"movie_graph.txt";
+ int n_ep=params1[0], n_st=params1[1], n_rep=params1[2];
+
+ //...............................
+
+ for(int i=0; i 0){
+
+ Results = new double[n_rep];
+ Counters = new int[n_rep];
+
+ }else {
+
+ Results = NULL;
+ Counters = NULL;
+ std::cout << " Number of reruns should be positive! " << std::endl;
+ return 0;
+
+ }
+ //...............................
+ srand(time(0));
+ //srandomdev();
+
+ {
+ double r=0;
+ for (int j=0; j<100; j++){
+
+
+
+ r = rand()/(double)(pow(2.,31)-1.);
+ std::cout << r << " ";
+ }
+ std::cout << "\n ";
+ }
+ //random initiation of starting parameters
+
+ if (range > 0.){
+
+ for (int i=0; i < 5; i++){
+
+ if (params0[i]==-100.){
+
+ double r1 = (rand()/(double)(pow(2.,31)-1.));
+ double r2 = (rand()/(double)(pow(2.,31)-1.));
+ double sign = 1.;
+
+ if(r1 > 0.5){
+ sign=-1.;
+ }
+
+ params0[i] = sign*r2*range;
+
+ std::cout << par_names0[i] << ": " << params0[i] << std::endl;
+ }
+ }
+
+ }
+
+
+ double T_start=params2[0], T_end=params2[1], Target_rejection=params2[3], starting_jump=params2[4];
+ int Annealing_repeats = (int) params2[2];
+
+
+ dispatch_group_t group = dispatch_group_create();
+
+ //.............................
+ multi_annealing(group, un, CustomQueues, T_start, T_end, Target_rejection, Annealing_repeats,
+ starting_jump, Results, Counters, params0, Annealing_repeats);
+
+ //dispatch_group_wait(group, DISPATCH_TIME_FOREVER);
+ dispatch_release(group);
+ //.............................
+
+
+ // stop timer
+ gettimeofday(&t2, NULL);
+
+ // compute and print the elapsed time in millisec
+ elapsedTime = (t2.tv_sec - t1.tv_sec) * 1000.0; // sec to ms
+ elapsedTime += (t2.tv_usec - t1.tv_usec) / 1000.0; // us to ms
+ elapsedTime /= 1000.;
+ cout << elapsedTime << " seconds \n .....(" << elapsedTime/60. << " minutes)\n\n";
+
+ //.....................
+
+ for(int i=0; i 0){
+
+ delete [] Results;
+ delete [] Counters;
+
+ }
+
+ return 0;
+
+
+
+}
+
Property changes on: SwiftApps/SciColSim/optimizer.cpp
___________________________________________________________________
Added: svn:executable
+ *
From wilde at ci.uchicago.edu Mon Jan 23 13:13:49 2012
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Mon, 23 Jan 2012 13:13:49 -0600 (CST)
Subject: [Swift-commit] r5513 - SwiftApps/SciColSim
Message-ID: <20120123191349.DA9199CFA1@svn.ci.uchicago.edu>
Author: wilde
Date: 2012-01-23 13:13:49 -0600 (Mon, 23 Jan 2012)
New Revision: 5513
Modified:
SwiftApps/SciColSim/Makefile
SwiftApps/SciColSim/optimizer.cpp
Log:
Fix bug in original optimizer which caused the inner loop of
multi_loss() to be executed way too many times - N^2 instead of N -
(per EMAIL circa Oct 16 2011):
> Looking deeper, I think the bug was simply that the first for() statement
> should have enclosed just the parameter setting loop. In other words, the top
> of multi_loss should start with:
>
> for(int i=0; i for(int j=0; j<5; j++){
> un[i]->set_parameter(params[j],j);
> }
> }
>
> Then the real work is done by the next loop:
> for(i=0; iset_parameter(params[j],j);
}
+ }
+
+ for(int i=0; ievolve_to_target_and_save(istart, iend, Results, Counters);
- });
+ un[i]->evolve_to_target_and_save(istart, iend, Results, Counters);
+ });
- istart += step;
- iend = min(istart+step,N);
+ std::cout << "queued: i=" << i << " N=" << N << " istart=" << istart << " iend=" << iend << "\n";
+ istart += step;
+ iend = min(istart+step,N);
- }
}
dispatch_group_wait(group, DISPATCH_TIME_FOREVER);
//dispatch_release(group);
From wilde at ci.uchicago.edu Mon Jan 23 15:05:03 2012
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Mon, 23 Jan 2012 15:05:03 -0600 (CST)
Subject: [Swift-commit] r5515 - SwiftApps/SciColSim
Message-ID: <20120123210503.EDCC09CC94@svn.ci.uchicago.edu>
Author: wilde
Date: 2012-01-23 15:05:03 -0600 (Mon, 23 Jan 2012)
New Revision: 5515
Added:
SwiftApps/SciColSim/optimizer.orig-mac.cpp
Log:
Add cpp file for orginal Mac Grand Central version with only necessary logic fixes beyond original.
Added: SwiftApps/SciColSim/optimizer.orig-mac.cpp
===================================================================
--- SwiftApps/SciColSim/optimizer.orig-mac.cpp (rev 0)
+++ SwiftApps/SciColSim/optimizer.orig-mac.cpp 2012-01-23 21:05:03 UTC (rev 5515)
@@ -0,0 +1,1673 @@
+//
+// main.cpp
+// optimizer
+//
+// Created by Andrey Rzhetsky on 4/11/11.
+// Copyright 2011 University of Chicago. All rights reserved.
+//
+
+#define Nworkers 24
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include
+#include
+#include
+#include
+
+#include
+#include
+
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include
+#include
+#include
+
+#define BOOST_MATH_OVERFLOW_ERROR_POLICY ignore_error
+#define BOOST_MATH_DISCRETE_QUANTILE_POLICY real
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#define INT_INFINITY 2147483647
+
+#define FIX_VARIABLES 1
+
+using namespace boost;
+using namespace std;
+using namespace boost::numeric::ublas;
+
+static int max_dist=0;
+
+typedef boost::adjacency_matrix Graph;
+typedef std::pair Edge;
+typedef boost::graph_traits GraphTraits;
+typedef boost::numeric::ublas::triangular_matrix prob;
+typedef boost::numeric::ublas::triangular_matrix pathlength;
+typedef boost::graph_traits::vertex_descriptor vertex_descriptor;
+
+namespace std {
+ using ::time;
+}
+
+static int var_fixed[5] = {1, 0, 1, 1, 0};
+
+typedef boost::minstd_rand base_generator_type;
+typedef adjacency_list < listS, vecS, directedS,
+no_property, property < edge_weight_t, int > > graph_t;
+typedef graph_traits < graph_t >::vertex_descriptor vertex_descriptor;
+typedef graph_traits < graph_t >::edge_descriptor edge_descriptor;
+
+
+//================================================
+string strDouble(double number)
+{
+ stringstream ss;//create a stringstream
+ ss << number;//add number to the stream
+ return ss.str();//return a string with the contents of the stream
+}
+
+//================================================
+
+double gaussian(double sigma)
+{
+ double GaussNum = 0.0;
+ int NumInSum = 10;
+ for(int i = 0; i < NumInSum; i++)
+ {
+ GaussNum += ((double)rand()/(double)RAND_MAX - 0.5);
+ }
+ GaussNum = GaussNum*sqrt((double)12/(double)NumInSum);
+
+
+ return GaussNum;
+
+}
+
+
+
+//=================================================
+double diffclock(clock_t clock1,clock_t clock2)
+{
+ double diffticks=clock1-clock2;
+ double diffms=(diffticks)/CLOCKS_PER_SEC;
+ return diffms;
+}
+
+//================================================
+//================================================================
+double get_new_x(double x, double dx){
+
+ double new_x;
+ // boost::variate_generator > uni(generator, uni_dist);
+ double r = rand()/(double)(pow(2.,31)-1.);
+
+ if (r > 0.5){
+ new_x = x + rand()*dx/(double)(pow(2.,31)-1.);
+ } else {
+ new_x = x - rand()*dx/(double)(pow(2.,31)-1.);
+ }
+
+ return new_x;
+
+}
+
+
+//===============================================
+string string_wrap(string ins, int mode){
+
+ std::ostringstream s;
+
+ switch(mode){
+ case 0:
+ s << "\033[1;29m" << ins << "\033[0m";
+ break;
+ case 1:
+ s << "\033[1;34m" << ins << "\033[0m";
+ break;
+ case 2:
+ s << "\033[1;44m" << ins << "\033[0m";
+ break;
+ case 3:
+ s << "\033[1;35m" << ins << "\033[0m";
+ break;
+ case 4:
+ s << "\033[1;33;44m" << ins << "\033[0m";
+ break;
+ case 5:
+ s << "\033[1;47;34m" << ins << "\033[0m";
+ break;
+ case 6:
+ s << "\033[1;1;31m" << ins << "\033[0m";
+ break;
+ case 7:
+ s << "\033[1;1;33m" << ins << "\033[0m";
+ break;
+ case 8:
+ s << "\033[1;1;43;34m" << ins << "\033[0m";
+ break;
+ case 9:
+ s << "\033[1;1;37m" << ins << "\033[0m";
+ break;
+ case 10:
+ s << "\033[1;30;47m" << ins << "\033[0m";
+ break;
+ default:
+ s << ins;
+ }
+
+ return s.str();
+}
+
+
+//===============================================
+string wrap_double(double val, int mode){
+
+ std::ostringstream s;
+ s << string_wrap(strDouble(val),mode);
+
+ return s.str();
+}
+
+
+
+//===============================================
+const
+string i2string(int i){
+
+ std::ostringstream s;
+ s << "worker"
+ << lexical_cast(i);
+
+ return s.str();
+
+}
+
+//===============================================
+char* i2char(int i){
+
+ std::ostringstream s;
+ s << "worker"
+ << lexical_cast(i);
+
+ char* a=new char[s.str().size()+1];
+ memcpy(a,s.str().c_str(), s.str().size());
+
+ return a;
+}
+
+//================================================
+class Universe {
+
+private:
+
+ double alpha_i;
+ double alpha_m;
+ double beta;
+ double gamma;
+ double delta;
+
+ double TargetNovelty;
+ double CumulativeRelativeLoss;
+ double CRLsquare;
+ string id;
+
+
+ int N_nodes;
+ int M_edges;
+
+ int N_epochs;
+ int N_steps;
+ int N_repeats;
+
+ int current_epoch;
+ double current_loss;
+ int current_repeat;
+ double current_novelty;
+
+ int mode_identify_failed;
+ int verbose_level; // 0 is silent, higher is more
+
+ double k_max;
+
+ graph_t Full_g;
+
+ double **Prob;
+ double **Tried;
+ double **Dist;
+ double **Final;
+ double **EdgeIndex;
+ double *Rank;
+
+ base_generator_type generator;
+ boost::uniform_real<> uni_dist;
+ boost::geometric_distribution geo;
+
+public:
+
+
+
+ //====== Constructor ======
+ Universe(const std::string FileToOpen, int Epochs, int Steps, int Repeats, int identify_failed, double target, const std::string idd)
+ {
+ //typedef array_type2::index index2;
+
+
+ std::ifstream inFile;
+ //string line;
+
+ //-------------------------------
+
+ base_generator_type gene(42u);
+ generator = gene;
+ generator.seed(static_cast(std::time(0)));
+ boost::uniform_real<> uni_d(0,1);
+ uni_dist = uni_d;
+
+ //--------------------------------
+
+ int i, k;
+ int x, y;
+ Edge* edge_array_mine;
+ int num_arcs_mine, num_nodes_mine;
+ int* weights_mine;
+
+ TargetNovelty = target;
+ CumulativeRelativeLoss = 0.;
+ CRLsquare = 0.;
+
+
+ N_epochs = Epochs;
+ N_steps = Steps;
+ N_repeats = Repeats;
+
+ current_epoch = 0;
+ current_loss = 0.;
+ current_repeat = 0;
+
+ id = idd;
+
+ verbose_level = 1;
+
+ mode_identify_failed = identify_failed;
+
+
+ //-------------------------------
+ // The first pass though file with the graph
+ inFile.open(FileToOpen.c_str());
+ if (inFile.fail()) {
+ cout << "Unable to open file";
+ exit(1); // terminate with error
+ }else {
+
+ if (verbose_level > 2){
+ std::cout << " Opened <" << FileToOpen << ">"<> x;
+ inFile >> y;
+
+ if (verbose_level > 2){
+ std::cout << " x: " << x;
+ std::cout << " y: " << y << std::endl;
+ }
+
+ if (i==0){
+ N_nodes=x;
+ M_edges=y;
+ break;
+ }
+ i++;
+
+
+ }
+ inFile.close();
+
+ if (verbose_level == 2){
+ std::cout << N_nodes << " nodes, " << M_edges << " edges"< 2){
+ std::cout << " Opened <" << FileToOpen << ">"<> x && inFile >>y) {
+ if (i > 0) {
+ Final[x][y]=1.;
+ Final[y][x]=1.;
+
+
+ if (verbose_level == 2){
+ std::cout << ".";
+ }
+ }
+ i++;
+
+ }
+ if (verbose_level == 2){
+ std::cout << std::endl;
+ }
+ inFile.close();
+
+ k=0;
+ for (int i=0; i 0.){
+ EdgeIndex[i][j]=k;
+ k++;
+ }
+ }
+ }
+
+
+
+ //+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ // create graph -- hopefully, we can keep it, just modifying edge weights
+
+
+ edge_array_mine = new Edge[2*M_edges];
+ num_arcs_mine = 2*M_edges;
+ num_nodes_mine = N_nodes;
+ weights_mine = new int[2*M_edges];
+ for (int i=0; i<2*M_edges; i++){ weights_mine[i]=1;}
+
+ k=0;
+ for(int i=0; i0.){
+ edge_array_mine[2*k] =Edge(i,j);
+ edge_array_mine[2*k+1]=Edge(j,i);
+ k++;
+ }
+ }
+ }
+ graph_t g(edge_array_mine, edge_array_mine + num_arcs_mine, weights_mine, num_nodes_mine);
+
+ Full_g = g;
+ delete edge_array_mine;
+ delete weights_mine;
+
+ //===========================================================================
+ std::vector p(num_edges(Full_g));
+ std::vector d(num_edges(Full_g));
+ edge_descriptor s;
+ boost::graph_traits::vertex_descriptor u, v;
+
+ for (int i=0; i 0.){
+ u = vertex(i, Full_g);
+ v = vertex(j, Full_g);
+ remove_edge(u,v,Full_g);
+ remove_edge(v,u,Full_g);
+
+ }
+ }
+ }
+
+
+ }
+
+
+ //=====================================================================
+ int sample_failed_number(double pfail){
+
+ //boost::geometric_distribution geo(pfail);
+ //boost::variate_generator > geom(generator, geo);
+
+ double r, u, g;
+
+ r=0.;
+ for(int i=0; i=3){
+ std::cout << id << " failed " << r << std::endl;
+ }
+ return r;
+
+ }
+
+ //=============================================
+ double get_target(void){
+ return TargetNovelty;
+ }
+
+ //=============================================
+ void set_target(double target){
+ TargetNovelty=target;
+ }
+
+ //=============================================
+ int sample(){
+
+ //boost::variate_generator > uni(generator, uni_dist);
+ // double r = uni(), Summa = 0.;
+
+
+
+ double r = rand(), Summa = 0.;
+ r /= (double)(pow(2.,31)-1.);
+ int result = 0;
+ int finished = 0;
+
+ if (verbose_level==4){
+ std::cout << id << " sampled " << r << std::endl;
+ }
+
+ for(int i=0; i r){
+
+ Tried[i][j]+=1.;
+
+ if (Final[i][j] > 0.){
+ result = 1;
+ }
+ finished = 1;
+ }
+ }
+ }
+
+ return result;
+
+ }
+
+ //===============================
+ void update_current_graph(void){
+
+ std::vector p(num_edges(Full_g));
+ std::vector d(num_edges(Full_g));
+ edge_descriptor s;
+ boost::graph_traits::vertex_descriptor u, v;
+
+ //property_map::type weightmap = get(edge_weight, Full_g);
+ for (int i=0; i 0. && Tried[i][j]>0){
+ //s = edge(i, j, Full_g);
+ boost::graph_traits::edge_descriptor e1,e2;
+ bool found1, found2;
+ u = vertex(i, Full_g);
+ v = vertex(j, Full_g);
+ tie(e1, found1) = edge(u, v, Full_g);
+ tie(e2, found2) = edge(v, u, Full_g);
+ if (!found1 && !found2){
+ add_edge(u,v,1,Full_g);
+ add_edge(v,u,1,Full_g);
+ }
+
+ }
+ }
+
+ }
+ }
+
+ //===============================
+ void update_distances(void){
+ // put shortest paths to the *Dist[][]
+ std::vector p(num_vertices(Full_g));
+ std::vector d(num_vertices(Full_g));
+ vertex_descriptor s;
+
+
+ // put shortest paths to the *Dist[][]
+ for (int j=0; j 0.){
+ s = vertex(j, Full_g);
+ dijkstra_shortest_paths(Full_g, s, predecessor_map(&p[0]).distance_map(&d[0]));
+
+ //std::cout <<" Vertex "<< j << std::endl;
+ graph_traits < graph_t >::vertex_iterator vi, vend;
+
+ for (boost::tie(vi, vend) = vertices(Full_g); vi != vend; ++vi) {
+
+ if (p[*vi]!=*vi){
+ Dist[*vi][j]=d[*vi];
+ Dist[j][*vi]=d[*vi];
+
+ if (Dist[*vi][j]>max_dist){
+ max_dist=Dist[*vi][j];
+ }
+
+
+ } else {
+ Dist[*vi][j]=-1.;
+ Dist[j][*vi]=-1.;
+ }
+ }
+ }
+
+ }
+
+
+ }
+
+ //======================================================
+ void update_ranks(void){
+
+ for(int i=0; i0. && Final[i][j] >0.){
+ Rank[i]++;
+ Rank[j]++;
+ }
+ }
+ }
+
+ }
+
+ //====================================================================
+ void set_world(double a_i, double a_m, double b, double g, double d){
+
+ alpha_i=a_i;
+ alpha_m=a_m;
+ gamma=g;
+ beta=b;
+ delta=d;
+
+ }
+
+ //====================================================================
+ void reset_world(){
+
+ //====================================================
+ std::vector p(num_edges(Full_g));
+ std::vector d(num_edges(Full_g));
+ edge_descriptor s;
+ boost::graph_traits::vertex_descriptor u, v;
+
+
+ for (int i=0; i 0. && Tried[i][j] > 0){
+ u = vertex(i, Full_g);
+ v = vertex(j, Full_g);
+ remove_edge(u,v,Full_g);
+ remove_edge(v,u,Full_g);
+
+ }
+ }
+ }
+
+ //==================================================
+
+ current_loss=0;
+ current_epoch=0;
+ current_repeat++;
+ current_novelty=0;
+
+ for(int i = 0; i < N_nodes; ++i) {
+ Rank[i]=0.;
+ for(int j = 0; j < N_nodes; ++j) {
+ Prob[i][j]=0.;
+ Dist[i][j]=-1.;
+ Tried[i][j]=0.;
+ }
+ }
+ }
+
+
+ //==============================================
+ void show_parameters(void){
+
+ std::cout << "Parameters: "
+ << alpha_i << " "
+ << alpha_m << " | "
+ << beta << " "
+ << gamma << " | "
+ << delta << std::endl;
+
+ }
+
+
+
+ //===============================================
+ string file_name(){
+
+ std::ostringstream s;
+ s << "world_"
+ << lexical_cast(alpha_i) << "_"
+ << lexical_cast(alpha_m) << "_"
+ << lexical_cast(beta) << "_"
+ << lexical_cast(gamma) << "_"
+ << lexical_cast(delta) << "_"
+ << lexical_cast(N_epochs) << "_"
+ << lexical_cast(N_steps) << "_"
+ << lexical_cast(N_repeats) << ".txt";
+
+ return s.str();
+
+ }
+
+
+
+
+ //=================================================
+ void set_verbose(int verbose){
+
+ verbose_level = verbose;
+ }
+
+
+ //=============================================================
+ void update_probabilities(void){
+
+
+ //=========================
+ // Compute sampling probabilities
+ // first pass: \xi_i,j
+ for(int i=0; i 0.){
+
+ double k = Dist[i][j];
+ if (k >= k_max){
+ k = k_max-1;
+ }
+
+ bg = beta * log(k/k_max) + gamma * log(1. - k/k_max);
+
+ } else {
+ bg = delta;
+ }
+
+ Prob[i][j] = exp(Prob[i][j] + bg);
+ }
+ }
+
+
+ // second pass: sum
+ double Summa = 0.;
+
+ for(int i=0; i0. && Final[i][j]>0.){
+ novel+=1.;
+ }
+ }
+ }
+
+ }
+ //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ else {
+
+ double pfail=0.;
+ int n_failed;
+ //, n_check = 0;
+
+ for(int i=0; i0. && Final[i][j]>0.){
+ novel+=1.;
+ }
+ }
+ }
+ }
+
+ current_novelty = novel;
+
+
+ //^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ if (verbose_level == 2){
+ std::cout << (current_repeat+1) << " epoch=" << (current_epoch+1)
+
+ << " cost=" << cost
+ << " novel=" << novel
+ << " rel_loss=" << cost/novel
+ << std::endl;
+ }
+
+ current_epoch++;
+ }
+
+
+ //====== Destructor ======
+ ~Universe(){
+
+ delete_2Dmatrix(Final, N_nodes);
+ delete_2Dmatrix(Dist, N_nodes);
+ delete_2Dmatrix(Tried, N_nodes);
+ delete_2Dmatrix(Prob, N_nodes);
+ delete_2Dmatrix(EdgeIndex, N_nodes);
+ delete_1Dmatrix(Rank);
+ }
+
+ //================================================
+ // Allocate memory
+ double** allocate_2Dmatrix(int N, int M)
+ {
+ double **pointer;
+
+ if (verbose_level == 2){
+ std::cout<< "["< 0){
+
+ pointer = new double[N];
+
+ }else {
+
+ pointer = NULL;
+ }
+
+ return pointer;
+
+ }
+
+ //==============================================
+ // De-Allocate memory to prevent memory leak
+ void delete_2Dmatrix(double **pointer, int N){
+
+ if (pointer != NULL){
+
+ for (int i = 0; i < N; ++i){
+ delete [] pointer[i];
+ }
+ delete [] pointer;
+ }
+ }
+ //====================
+ void delete_1Dmatrix(double *pointer){
+
+ delete [] pointer;
+ }
+
+ //===========================================
+ double get_rel_loss(){
+
+ return CumulativeRelativeLoss ;
+ }
+
+ //===========================================
+ double get_rel_loss_err(){
+
+ return CRLsquare ;
+ }
+
+
+
+ //==================================================================================
+ void evolve_to_target_and_save(int istart, int iend, double* storage, int* counters){
+
+ double ALOT=100000000000.;
+
+ reset_world();
+
+ for (int k = istart; k < iend; k++){
+
+
+ for(int i=0; i< N_epochs && current_novelty < TargetNovelty; i++){
+ update_world();
+ }
+
+ storage[k]=current_loss/current_novelty;
+ counters[k]=1;
+
+
+ reset_world();
+ }
+
+ }
+ //==============================================
+ int get_reruns(void){
+ return N_repeats;
+ }
+
+ //==============================================
+ double get_parameter(int i){
+
+ switch(i){
+ case 0:
+ return alpha_i;
+ case 1:
+ return alpha_m;
+ case 2:
+ return beta;
+ case 3:
+ return gamma;
+ case 4:
+ return delta;
+ default:
+
+ std::cout << "Erroneous parameter id!!!!\n\n\n";
+ return 0.;
+ }
+ }
+
+
+ //==============================================
+ void evolve_to_target(){
+
+ reset_world();
+ if (beta < -1. || gamma < -1.){
+ CumulativeRelativeLoss = 100000000000.;
+ CRLsquare = 0.;
+ return;
+ }
+
+
+ for (int k=0; k< N_repeats; k++){
+
+
+ for(int i=0; i 4) {return 0;}
+
+ else {
+
+ switch(position){
+ case 0:
+ alpha_i=value;
+ return 1;
+ case 1:
+ alpha_m=value;
+ return 1;
+ case 2:
+ beta=value;
+ return 1;
+ case 3:
+ gamma=value;
+ return 1;
+ case 4:
+ delta=value;
+ return 1;
+ }
+
+ }
+
+ return 0;
+ }
+
+
+ //=================================================================
+ void try_annealing(double starting_jump, int iterations,
+ double temp_start, double temp_end, double target_rejection){
+
+ double dx[5]={0.,0.,0.,0.,0};
+ double x[5]={0.,0.,0.,0.,0};
+ double rejection[5]={0., 0., 0., 0., 0.};
+ double curr_x, curr_err, x_tmp;
+ double temperature;
+ double ratio, r;
+ int cycle=10;
+ boost::variate_generator > uni(generator, uni_dist);
+
+ // set up parameter for annealing
+
+ x[0]=alpha_i;
+ x[1]=alpha_m;
+ x[2]=beta;
+ x[3]=gamma;
+ x[4]=delta;
+
+ for(int i=0;i<5;i++){
+ dx[i] = starting_jump;
+ }
+
+ // establish the current value
+
+ //..........................................
+ evolve_to_target();
+ std::cout << CumulativeRelativeLoss << " +- " << CRLsquare << std::endl;
+
+ curr_x = CumulativeRelativeLoss;
+ curr_err = CRLsquare;
+ CumulativeRelativeLoss = 0;
+ CRLsquare = 0;
+ //...........................................
+
+ // optimization cycle
+ for(int i=0; i 0){
+
+ for (int k=0; k<5; k++){
+
+ rejection[k]/=(double)cycle;
+ if (rejection[k] > 0){
+ dx[k] = dx[k]/(rejection[k]/target_rejection);
+ rejection[k]=0.;
+ }
+ else{
+ dx[k]*=2.;
+ }
+ std::cout << dx[k] << " ";
+ }
+ std::cout << std::endl;
+ }
+
+
+ for (int j=0; j<5; j++){
+
+ // get new value of x[j]
+ x_tmp = get_new_x(x[j],dx[j]);
+
+
+
+ //.............................................
+ set_parameter(x_tmp, j);
+
+
+ evolve_to_target();
+
+ std::cout << std::endl << "......... " << std::endl;
+ std::cout << "Trying... " << CumulativeRelativeLoss << " +- " << CRLsquare << std::endl;
+
+ ratio = min(1.,exp(-(CumulativeRelativeLoss-curr_x)/temperature));
+ r = uni();
+ std::cout << r << " vs " << ratio << std::endl;
+
+ if (r > ratio){
+
+ std::cout << string_wrap(id, 4) <<" "<< (i+1) << ","<< (j)
+ <<" "<< (i+1) << " Did not accept "
+ << x_tmp << "(" << j << ")" << std::endl;
+ std::cout << alpha_i << " "<< alpha_m << " "
+ << beta << " " << gamma << " "
+ << delta << " " << std::endl;
+ set_parameter(x[j], j);
+ CumulativeRelativeLoss = 0;
+ CRLsquare = 0;
+
+ rejection[j]+=1.;
+ }
+
+ else {
+
+ curr_x = CumulativeRelativeLoss;
+ curr_err = CRLsquare;
+ x[j] = x_tmp;
+ CumulativeRelativeLoss = 0;
+ CRLsquare = 0;
+ std::cout << (i+1) << string_wrap((string) " Rejection counts: ", 8)
+ << wrap_double(rejection[0],2)
+ << " "<< wrap_double(rejection[1], 7) << " "
+ << wrap_double(rejection[2],5) << " " << wrap_double(rejection[2],9) << " "
+ << wrap_double(rejection[4],6) << " "
+ << std::endl << std::endl;
+
+ std::cout << string_wrap(id, 4) <<" "<< (i+1) <<","<< (j)
+ <<" "
+ << string_wrap((string) "***** Did accept! ", 3)
+ << wrap_double(alpha_i,2)
+ << " "<< wrap_double(alpha_m, 7) << " "
+ << wrap_double(beta,5) << " "
+ << wrap_double(gamma,9) << " "
+ << wrap_double(delta,6) << " "
+ << std::endl << std::endl;
+
+ }
+ //........................................................
+
+ }
+
+ }
+
+ }
+
+
+};
+
+//============================================================
+
+std::pair multi_loss(dispatch_group_t group,
+ Universe* un[],
+ dispatch_queue_t* CustomQueues,
+ double* Results,
+ int* Counters,
+ double* params){
+
+ int N = un[0]->get_reruns();
+ int step = (int)(double)N/(double)(Nworkers);
+ int istart=0;
+ int iend = istart+step;
+
+ double Loss=0., LossSquare=0.;
+
+ for(int i=0; iset_parameter(params[j],j);
+ }
+ }
+
+
+ for(int i=0; ievolve_to_target_and_save(istart, iend, Results, Counters);
+ });
+
+ std::cout << "queued: i=" << i << " N=" << N << " istart=" << istart << " iend=" << iend << "\n";
+ istart += step;
+ iend = min(istart+step,N);
+
+ }
+ dispatch_group_wait(group, DISPATCH_TIME_FOREVER);
+ //dispatch_release(group);
+
+ for (int i=0; i Res;
+ Res.first=Loss;
+ Res.second=two_std;
+
+ return Res;
+
+
+}
+//============================================================
+
+
+//============================================================
+void multi_annealing( dispatch_group_t group,
+ Universe* un[],
+ dispatch_queue_t* CustomQueues,
+ double T_start, double T_end,
+ double Target_rejection,
+ int Annealing_repeats,
+ double starting_jump,
+ double* Results,
+ int* Counters,
+ double* params0,
+ double annealing_cycles){
+ //.................................
+ // re-implement annealing
+
+ double dx[5]={0.,0.,0.,0.,0};
+ double x[5]={0.,0.,0.,0.,0};
+ double rejection[5]={0., 0., 0., 0., 0.};
+ double curr_x, curr_err, x_tmp;
+ double temperature;
+ double ratio, r;
+ int cycle=10;
+ //boost::variate_generator > uni(generator, uni_dist);
+
+ // set up parameter for annealing
+
+ x[0]=params0[0];
+ x[1]=params0[1];
+ x[2]=params0[2];
+ x[3]=params0[3];
+ x[4]=params0[4];
+
+ for(int i=0;i<5;i++){
+ dx[i] = starting_jump;
+ }
+
+ // establish the current value
+ std::pairRes;
+
+ Res = multi_loss(group, un, CustomQueues, Results, Counters, x);
+ std::cout << Res.first << " +- " << Res.second << std::endl;
+
+ curr_x = Res.first;
+ curr_err = Res.second;
+
+ // optimization cycle
+
+ for(int i=0; i 0){
+
+ for (int k=0; k<5; k++){
+ rejection[k]/=(double)cycle;
+
+ if (rejection[k] > 0){
+ dx[k] = dx[k]/(rejection[k]/Target_rejection);
+ rejection[k]=0.;
+ }
+ else{
+ dx[k]*=2.;
+ }
+ std::cout << dx[k] << " ";
+ }
+ std::cout << std::endl;
+ }
+
+
+ for (int j=0; j<5; j++){
+
+ ///////////////////////////////
+ if (FIX_VARIABLES==0 || var_fixed[j]==0){
+
+
+
+ // get new value of x[j]
+ double x_hold=x[j];
+ x_tmp = get_new_x(x[j],dx[j]);
+ x[j]=x_tmp;
+
+ std::cout << wrap_double(x_tmp,10) << " " << wrap_double(j,9) << "\n\n";
+ //=======================================
+ //.............................................
+ for(int w=0; wset_parameter(x_tmp, j);
+ }
+
+
+ Res = multi_loss(group, un, CustomQueues, Results, Counters, x);
+ std::cout << Res.first << " +- " << Res.second << std::endl;
+
+ ratio = min(1.,exp(-(Res.first-curr_x)/temperature));
+ r = rand()/(double)(pow(2.,31)-1.);
+ std::cout << r << " vs " << ratio << std::endl;
+
+ double ALOT=100000000000.;
+
+ if (Res.first < ALOT)
+ {
+ ofstream filestr;
+
+ filestr.open ("best_opt_some.txt", ofstream::app);
+
+ // >> i/o operations here <<
+ filestr << un[0]->get_target() << ","
+ << Res.first
+ << "," << un[0]->get_parameter(0)
+ << "," << un[0]->get_parameter(1)
+ << "," << un[0]->get_parameter(2)
+ << "," << un[0]->get_parameter(3)
+ << "," << un[0]->get_parameter(4) << "," << Res.second << ",\n";
+
+ filestr.close();
+
+
+ filestr.open ("max_dist.txt", ofstream::app);
+
+ // >> i/o operations here <<
+ filestr << max_dist << ",\n";
+
+ filestr.close();
+
+ }
+
+
+ if (r > ratio){
+
+ std::cout << " "<< (i+1) << ","<< (j)
+ <<" "<< (i+1) << " Did not accept "
+ << x_tmp << "(" << j << ")" << std::endl;
+ std::cout << un[0]->get_parameter(0)
+ << " " << un[0]->get_parameter(1)
+ << " " << un[0]->get_parameter(2)
+ << " " << un[0]->get_parameter(3)
+ << " " << un[0]->get_parameter(4) << " " << std::endl;
+
+ x[j]=x_hold;
+ for(int w=0; wset_parameter(x[j], j);
+ }
+
+
+ //set_parameter(x[j], j);
+ rejection[j]+=1.;
+ }
+
+ else {
+
+ curr_x = Res.first;
+ curr_err = Res.second;
+ x[j] = x_tmp;
+
+ for(int w=0; wset_parameter(x[j], j);
+ }
+
+ std::cout << (i+1) << string_wrap((string) " Rejection counts: ", 8)
+ << wrap_double(rejection[0],2) << " "
+ << wrap_double(rejection[1],7) << " "
+ << wrap_double(rejection[2],5) << " "
+ << wrap_double(rejection[3],9) << " "
+ << wrap_double(rejection[4],6) << " "
+ << std::endl << std::endl;
+
+ std::cout << " "<< (i+1) <<","<< (j)
+ <<" "
+ << string_wrap((string) "***** Did accept! ", 3)
+ << wrap_double(un[0]->get_parameter(0),2) << " "
+ << wrap_double(un[0]->get_parameter(1),7) << " "
+ << wrap_double(un[0]->get_parameter(2),5) << " "
+ << wrap_double(un[0]->get_parameter(3),9) << " "
+ << wrap_double(un[0]->get_parameter(4),6) << " "
+ << std::endl << std::endl;
+
+
+
+ }
+ //........................................................
+
+ }
+ }
+
+ }
+
+}
+
+
+
+//================================================
+int
+main(int argc, char* argv[])
+{
+
+ double params0[6] = {0., 0., 0., 0., 0., 0.2}, target=50., range;
+ string par_names0[6] = {"alpha_i", "alpha_m", "beta", "gamma", "delta", "target"};
+ string par_names1[4] = {"n_epochs", "n_steps", "n_reruns", "range"};
+ string par_names2[5] = {"T_start", "T_end", "Annealing_steps","Target_rejection","Starting_jump"};
+ string par_names3[5] = {"FREEZE_alpha_i", "FREEZE_alpha_m", "FREEZE_beta", "FREEZE_gamma", "FREEZE_delta"};
+ int params1[4] = {300, 50, 1000, 10};
+ int params3[5] = { 0, 0, 0, 0, 0};
+
+ // temperature_start, temperature_end, annealing_steps target_rejection Starting_jump
+ double params2[5] = {1, 0.001, 100, 0.3, 1.5};
+
+ int verbose_level = 2;
+ const std::string one="one", two="two";
+ static Universe* un[Nworkers];
+ static dispatch_queue_t CustomQueues[Nworkers];
+
+ static double* Results;
+ static int* Counters;
+
+ timeval t1, t2;
+ double elapsedTime;
+ // start timer
+ gettimeofday(&t1, NULL);
+
+
+ if (argc < 8) {
+ std::cout << "Usage: super_optimizer alpha_i alpha_m beta gamma delta target_innov [n_epochs n_steps n_reruns] [range] [verbose_level]\n";
+ std::cout << " [T_start T_end Annealing_steps Target_rejection Starting_jump]\n";
+ std::cout << " [FREEZE_alpha_i FREEZE_alpha_m FREEZE_beta FREEZE_gamma FREEZE_delta]\n";
+
+ system("pwd");
+
+
+ return(1);
+ }
+ else {
+ for (int nArg=0; nArg < argc; nArg++){
+ //std::cout << nArg << " " << argv[nArg] << std::endl;
+ if (nArg > 0 && nArg < 7){
+ params0[nArg-1]= atof(argv[nArg]);
+ std::cout << par_names0[nArg-1] << ": " << params0[nArg-1] << std::endl;
+ }
+ if (nArg > 6 && nArg < 11){
+ params1[nArg-7]= atoi(argv[nArg]);
+ std::cout << par_names1[nArg-7] << ": " << params1[nArg-7] << std::endl;
+ }
+ if (nArg == 11){
+ verbose_level = atoi(argv[nArg]);
+ std::cout << "verbose level: " << verbose_level << std::endl;
+ }
+ if (nArg > 11 && nArg < 17){
+ params2[nArg-12]= atof(argv[nArg]);
+ std::cout << par_names2[nArg-12] << ": " << params2[nArg-12] << std::endl;
+ }
+ if (nArg > 16 && nArg < 22){
+ params3[nArg-17]= atof(argv[nArg]);
+ var_fixed[nArg-17]= atof(argv[nArg]);
+ std::cout << par_names3[nArg-17] << ": " << var_fixed[nArg-17] << std::endl;
+ }
+
+
+ }
+
+ }
+
+ for (int j=0; j<5; j++){
+
+ cout << j << " | " << var_fixed[j] << " (fixed) \n";
+ }
+
+ target=params0[5];
+ range = (double)params1[3];
+ int identify_failed = 0;
+ char* filename= (char *)"movie_graph.txt";
+ int n_ep=params1[0], n_st=params1[1], n_rep=params1[2];
+
+ //...............................
+
+ for(int i=0; i 0){
+
+ Results = new double[n_rep];
+ Counters = new int[n_rep];
+
+ }else {
+
+ Results = NULL;
+ Counters = NULL;
+ std::cout << " Number of reruns should be positive! " << std::endl;
+ return 0;
+
+ }
+ //...............................
+ srand(time(0));
+ //srandomdev();
+
+ {
+ double r=0;
+ for (int j=0; j<100; j++){
+
+
+
+ r = rand()/(double)(pow(2.,31)-1.);
+ std::cout << r << " ";
+ }
+ std::cout << "\n ";
+ }
+ //random initiation of starting parameters
+
+ if (range > 0.){
+
+ for (int i=0; i < 5; i++){
+
+ if (params0[i]==-100.){
+
+ double r1 = (rand()/(double)(pow(2.,31)-1.));
+ double r2 = (rand()/(double)(pow(2.,31)-1.));
+ double sign = 1.;
+
+ if(r1 > 0.5){
+ sign=-1.;
+ }
+
+ params0[i] = sign*r2*range;
+
+ std::cout << par_names0[i] << ": " << params0[i] << std::endl;
+ }
+ }
+
+ }
+
+
+ double T_start=params2[0], T_end=params2[1], Target_rejection=params2[3], starting_jump=params2[4];
+ int Annealing_repeats = (int) params2[2];
+
+
+ dispatch_group_t group = dispatch_group_create();
+
+ //.............................
+ multi_annealing(group, un, CustomQueues, T_start, T_end, Target_rejection, Annealing_repeats,
+ starting_jump, Results, Counters, params0, Annealing_repeats);
+
+ //dispatch_group_wait(group, DISPATCH_TIME_FOREVER);
+ dispatch_release(group);
+ //.............................
+
+
+ // stop timer
+ gettimeofday(&t2, NULL);
+
+ // compute and print the elapsed time in millisec
+ elapsedTime = (t2.tv_sec - t1.tv_sec) * 1000.0; // sec to ms
+ elapsedTime += (t2.tv_usec - t1.tv_usec) / 1000.0; // us to ms
+ elapsedTime /= 1000.;
+ cout << elapsedTime << " seconds \n .....(" << elapsedTime/60. << " minutes)\n\n";
+
+ //.....................
+
+ for(int i=0; i 0){
+
+ delete [] Results;
+ delete [] Counters;
+
+ }
+
+ return 0;
+
+
+
+}
+
Property changes on: SwiftApps/SciColSim/optimizer.orig-mac.cpp
___________________________________________________________________
Added: svn:executable
+ *
From wilde at ci.uchicago.edu Mon Jan 23 15:49:09 2012
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Mon, 23 Jan 2012 15:49:09 -0600 (CST)
Subject: [Swift-commit] r5516 - SwiftApps/SciColSim
Message-ID: <20120123214909.6BB259CC94@svn.ci.uchicago.edu>
Author: wilde
Date: 2012-01-23 15:49:09 -0600 (Mon, 23 Jan 2012)
New Revision: 5516
Added:
SwiftApps/SciColSim/colortext.swift
Log:
temporary functions to format printed text in color (to match C++ output for optimizer).
Added: SwiftApps/SciColSim/colortext.swift
===================================================================
--- SwiftApps/SciColSim/colortext.swift (rev 0)
+++ SwiftApps/SciColSim/colortext.swift 2012-01-23 21:49:09 UTC (rev 5516)
@@ -0,0 +1,75 @@
+global string e33=@arg("e33");
+global string endEscape = "[0m";
+
+global int Black = 0;
+global int Blue = 1;
+global int BlackOnBlue = 2;
+global int Pink = 3;
+global int GoldOnBlue = 4;
+global int BlueOnGray = 5;
+global int Red = 6;
+global int Gold = 7;
+global int BlueOnGold = 8;
+global int Gray = 9;
+global int BlackOnGray = 10;
+
+# Black = 0;
+# Blue = 1;
+# BlackOnBlue = 2;
+# Pink = 3;
+# GoldOnBlue = 4;
+# BlueOnGray = 5;
+# Red = 6;
+# Gold = 7;
+# BlueOnGold = 8;
+# Gray = 9;
+# BlackOnGray = 10;
+
+global string colorCode[] = [
+ "[1;29m", # Black = 0;
+ "[1;34m", # Blue = 1;
+ "[1;44m", # BlackOnBlue = 2;
+ "[1;35m", # Pink = 3;
+ "[1;33;44m", # GoldOnBlue = 4;
+ "[1;47;34m", # BlueOnGray = 5;
+ "[1;1;31m", # Red = 6;
+ "[1;1;33m", # Gold = 7;
+ "[1;1;43;34m", # BlueOnGold = 8;
+ "[1;1;37m", # Gray = 9;
+ "[1;30;47m", # BlackOnGray = 10;
+];
+
+(string s) color(int c, string ins)
+{
+ s = @strcat(e33,colorCode[c],ins,e33,endEscape);
+}
+
+(string s) OLDcolor(int c, string ins)
+{
+ switch(c){
+ case 0:
+ s = @strcat(e33,"[1;29m",ins,e33,"[0m");
+ case 1:
+ s = @strcat(e33,"[1;34m",ins,e33,"[0m");
+ case 2:
+ s = @strcat(e33,"[1;44m",ins,e33,"[0m");
+ case 3:
+ s = @strcat(e33,"[1;35m",ins,e33,"[0m");
+ case 4:
+ s = @strcat(e33,"[1;33;44m",ins,e33,"[0m");
+ case 5:
+ s = @strcat(e33,"[1;47;34m",ins,e33,"[0m");
+ case 6:
+ s = @strcat(e33,"[1;1;31m",ins,e33,"[0m");
+ case 7:
+ s = @strcat(e33,"[1;1;33m",ins,e33,"[0m");
+ case 8:
+ s = @strcat(e33,"[1;1;43;34m",ins,e33,"[0m");
+ case 9:
+ s = @strcat(e33,"[1;1;37m",ins,e33,"[0m");
+ case 10:
+ s = @strcat(e33,"[1;30;47m",ins,e33,"[0m");
+ default:
+ s = ins;
+ }
+}
From lgadelha at ci.uchicago.edu Tue Jan 24 12:38:35 2012
From: lgadelha at ci.uchicago.edu (lgadelha at ci.uchicago.edu)
Date: Tue, 24 Jan 2012 12:38:35 -0600 (CST)
Subject: [Swift-commit] r5517 - provenancedb
Message-ID: <20120124183835.CA5DD9CC99@svn.ci.uchicago.edu>
Author: lgadelha
Date: 2012-01-24 12:38:35 -0600 (Tue, 24 Jan 2012)
New Revision: 5517
Added:
provenancedb/SPQL.g
Modified:
provenancedb/prov-init.sql
Log:
Minor changes
Added: provenancedb/SPQL.g
===================================================================
--- provenancedb/SPQL.g (rev 0)
+++ provenancedb/SPQL.g 2012-01-24 18:38:35 UTC (rev 5517)
@@ -0,0 +1,743 @@
+grammar ProvSQL;
+
+ at header {
+ import java.util.HashSet;
+ import java.util.HashMap;
+ import java.util.Iterator;
+ import org.jgrapht.*;
+ import org.jgrapht.alg.DijkstraShortestPath;
+ import org.jgrapht.graph.*;
+}
+
+ at members {
+ String selectClause = new String();
+ String fromClause = new String();
+ String whereClauseJoinExpressions = new String();
+ String whereClause = new String();
+ boolean hasWhereJoinExpression;
+ boolean hasWhereExpression = false;
+ static boolean hasCompareRunCall = false;
+ HashSet relations = new HashSet();
+ UndirectedGraph schemaGraph;
+ HashSet joinEdges;
+ static HashSet compareRunParams = new HashSet();;
+
+ // Ideally it could receive a DB schema in SQL and build the graph automatically
+ public static UndirectedGraph buildGraph() {
+ UndirectedGraph schemaGraph = new Multigraph