Add benchmark file

README.md

@@ -1,14 +1,36 @@
 # Running:
 
-For now, just modify the script in `paralleleureqa.jl`
-to your liking and run:
+You can run the performance benchmark with `./benchmark.sh`.
+
+Modify the search code in `paralleleureqa.jl` and `eureqa.jl` to your liking
+(see below for options). Then, in a new Julia file called
+`myfile.jl`, you can write:
+
+```julia
+include("paralleleureqa.jl")
+fullRun(10,
+    npop=100,
+    annealing=true,
+    ncyclesperiteration=1000,
+    fractionReplaced=0.1f0,
+    verbosity=100)
+```
+The first arg is the number of migration periods to run,
+with `ncyclesperiteration` determining how many generations
+per migration period.  `npop` is the number of population members.
+`annealing` determines whether to stay in exploration mode,
+or tune it down with each cycle. `fractionReplaced` is
+how much of the population is replaced by migrated equations each
+step.
+
 
-`julia --threads auto -O3 paralleleureqa.jl`
+Run it with threading turned on using:
+`julia --threads auto -O3 myfile.jl`
 
 ## Modification
 
 You can change the binary and unary operators in `eureqa.jl` here:
-```
+```julia
 const binops = [plus, mult]
 const unaops = [sin, cos, exp];
 ```
@@ -28,10 +50,6 @@ by either loading in a dataset, or modifying the definition of `y`.
 
 ### Hyperparameters
 
-Turn on annealing by setting the following in `paralleleureqa.jl`:
-
-`const annealing = true`
-
 Annealing allows each evolutionary cycle to turn down the exploration
 rate over time: at the end (temperature 0), it will only select solutions
 better than existing solutions.

benchmark.jl

@@ -0,0 +1,14 @@
+include("paralleleureqa.jl")
+fullRun(1,
+    npop=100,
+    annealing=true,
+    ncyclesperiteration=1000,
+    fractionReplaced=0.1f0,
+    verbosity=0)
+@time fullRun(3,
+    npop=100,
+    annealing=true,
+    ncyclesperiteration=1000,
+    fractionReplaced=0.1f0,
+    verbosity=0
+)

benchmark.sh

@@ -0,0 +1 @@
+julia --threads 8 -O3 benchmark.jl

eureqa.jl

@@ -35,6 +35,10 @@ const nbin = size(binops)[1]
 const nops = nuna + nbin
 const nvar = size(X)[2];
 
+function debug(verbosity, string...)
+    verbosity > 0 ? println(string...) : nothing
+end
+
 # Define a serialization format for the symbolic equations:
 mutable struct Node
     #Holds operators, variables, constants in a tree
@@ -241,7 +245,7 @@ end
 function scoreFunc(
         tree::Node,
         X::Array{Float32, 2},
-        y::Array{Float32, 1},
+        y::Array{Float32, 1};
         parsimony::Float32=0.1f0)::Float32
     try
         return MSE(evalTreeArray(tree, X), y) + countNodes(tree)*parsimony
@@ -341,8 +345,8 @@ function iterate(
     end
     
     try
-        beforeLoss = scoreFunc(prev, X, y, mult)
-        afterLoss = scoreFunc(tree, X, y, mult)
+        beforeLoss = scoreFunc(prev, X, y, parsimony=mult)
+        afterLoss = scoreFunc(tree, X, y, parsimony=mult)
         delta = afterLoss - beforeLoss
         probChange = exp(-delta/(T*alpha))
 
@@ -378,7 +382,7 @@ mutable struct PopMember
     score::Float32
     birth::Int32
     
-    PopMember(t) = new(t, scoreFunc(t, X, y, parsimony), round(Int32, 1e3*(time()-1.6e9))
+    PopMember(t) = new(t, scoreFunc(t, X, y, parsimony=parsimony), round(Int32, 1e3*(time()-1.6e9))
 )
 end
 
@@ -418,7 +422,7 @@ function iterateSample(pop::Population, T::Float32)::PopMember
     allstar = bestOfSample(pop)
     new = iterate(allstar.tree, T, X, y, alpha, parsimony)
     allstar.tree = new
-    allstar.score = scoreFunc(new, X, y, parsimony)
+    allstar.score = scoreFunc(new, X, y, parsimony=parsimony)
     allstar.birth = round(Int32, 1e3*(time()-1.6e9))
     return allstar
 end
@@ -441,7 +445,7 @@ function run(
         pop::Population,
         ncycles::Integer,
         annealing::Bool=false;
-        verbose::Integer=0
+        verbosity::Integer=0
         )::Population
     pop = deepcopy(pop)
 
@@ -452,12 +456,11 @@ function run(
         else
             pop = regEvolCycle(pop, 1.0f0)
         end
-        if verbose > 0 && (iT % verbose == 0)
-            # Get best 10 models from each evolution. Copy because we re-assign later.
+        if verbosity > 0 && (iT % verbosity == 0)
             bestPops = bestSubPop(pop)
             bestCurScoreIdx = argmin([bestPops.members[member].score for member=1:bestPops.n])
             bestCurScore = bestPops.members[bestCurScoreIdx].score
-            println(bestCurScore, " is the score for ", stringTree(bestPops.members[bestCurScoreIdx].tree))
+            debug(verbosity, bestCurScore, " is the score for ", stringTree(bestPops.members[bestCurScoreIdx].tree))
         end
     end
     return pop

paralleleureqa.jl

@@ -1,30 +1,31 @@
 include("eureqa.jl")
 
-println("Lets try to learn (x2^2 + cos(x3)) using regularized evolution from scratch")
 const nthreads = Threads.nthreads()
-println("Running with $nthreads threads")
-const npop = 300
-const annealing = true
-const ncyclesperiteration = 30000
-const fractionReplaced = 0.1
 
-function fullRun(niterations::Integer)
+function fullRun(niterations::Integer;
+                npop::Integer=300,
+                annealing::Bool=true,
+                ncyclesperiteration::Integer=3000,
+                fractionReplaced::Float32=0.1f0,
+                verbosity::Integer=0,
+               )
+    debug(verbosity, "Lets try to learn (x2^2 + cos(x3)) using regularized evolution from scratch")
+    debug(verbosity, "Running with $nthreads threads")
     # Generate random initial populations
     allPops = [Population(npop, 3) for j=1:nthreads]
     # Repeat this many evolutions; we collect and migrate the best
     # each time.
     for k=1:niterations
-
         # Spawn threads to run indepdent evolutions, then gather them
         @inbounds Threads.@threads for i=1:nthreads
-            allPops[i] = run(allPops[i], ncyclesperiteration, annealing, verbose=500)
+            allPops[i] = run(allPops[i], ncyclesperiteration, annealing, verbosity=verbosity)
         end
 
         # Get best 10 models from each evolution. Copy because we re-assign later.
         bestPops = deepcopy(Population([member for pop in allPops for member in bestSubPop(pop).members]))
         bestCurScoreIdx = argmin([bestPops.members[member].score for member=1:bestPops.n])
         bestCurScore = bestPops.members[bestCurScoreIdx].score
-        println(bestCurScore, " is the score for ", stringTree(bestPops.members[bestCurScoreIdx].tree))
+        debug(verbosity, bestCurScore, " is the score for ", stringTree(bestPops.members[bestCurScoreIdx].tree))
 
         # Migration
         for j=1:nthreads
@@ -36,4 +37,3 @@ function fullRun(niterations::Integer)
     end
 end
 
-fullRun(10)