# HG changeset patch
# User Steve Losh <steve@stevelosh.com>
# Date 1495632477 0
# Node ID 2ad170fcb59888bfe6625cc84dcbb5d1322750a4
# Parent  50011302bb369e8f1ce70bf556cce1a2b80d71bc
Better rule tree shuffling tests

diff -r 50011302bb36 -r 2ad170fcb598 plot-shuffle-rule-trees.gp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/plot-shuffle-rule-trees.gp	Wed May 24 13:27:57 2017 +0000
@@ -0,0 +1,31 @@
+set terminal png size 1200, 1200 linewidth 3
+
+set xtics font "Lucida Grande,18"
+set ytics font "Lucida Grande,18"
+
+
+set size 0.6, 0.8
+set bmargin at screen 0.18
+set tmargin at screen 0.9
+set lmargin at screen 0.1
+set rmargin at screen 0.95
+
+# set xlabel "Game" font "Lucida Grande,12"
+set ylabel "Ratio of shuffled node counts to unshuffled node count" font "Lucida Grande,18" offset -1
+
+
+set xtics rotate by -40 offset 0,-1
+
+set output 'plot-shuffling-rule-trees.png'
+set boxwidth 0.2 absolute
+set title "Effects of variable order shuffling on rule tree sizes" font "Lucida Grande,24"
+
+set xrange[0.2:8]
+set yrange[0.5:1.3]
+
+# Data columns: X Min 1stQuartile Median 3rdQuartile Max BoxWidth Titles
+
+# set bars 4.0
+set style fill empty
+plot 'data-shuffling-rule-trees' using 1:3:2:6:5:7:xticlabels(8) with candlesticks notitle whiskerbars, \
+  ''         using 1:4:4:4:4:7 with candlesticks lt -1 notitle
diff -r 50011302bb36 -r 2ad170fcb598 src/reasoners/zdd.lisp
--- a/src/reasoners/zdd.lisp	Tue May 23 19:54:56 2017 +0000
+++ b/src/reasoners/zdd.lisp	Wed May 24 13:27:57 2017 +0000
@@ -609,46 +609,49 @@
 (defparameter *i* nil)
 (defparameter *r* nil)
 
-;; (defparameter *rules* (scully.gdl::read-gdl "gdl/meier-grounded.gdl"))
-;; (defparameter *rules* (scully.gdl::read-gdl "gdl/kriegTTT_5x5-grounded.gdl"))
-;; (defparameter *rules* (scully.gdl::read-gdl "gdl/pennies-grounded.gdl"))
-;; (defparameter *rules* (scully.gdl::read-gdl "gdl/mastermind-grounded.gdl"))
-;; (defparameter *rules* (scully.gdl::read-gdl "gdl/mastermind448-grounded.gdl"))
-;; (defparameter *rules* (scully.gdl::read-gdl "gdl/montyhall-grounded.gdl"))
-;; (defparameter *rules* (scully.gdl::read-gdl "gdl/tictactoe-grounded.gdl"))
-;; (defparameter *rules* (scully.gdl::read-gdl "gdl/stratego-grounded.gdl"))
-;; (defparameter *rules* (scully.gdl::read-gdl "gdl/transit-grounded.gdl"))
-;; (defparameter *rules* (scully.gdl::read-gdl "gdl/vis_pacman3p-grounded.gdl"))
+; Data columns: X Min 1stQuartile Median 3rdQuartile Max BoxWidth Titles
 
-;; (defparameter *r* (make-zdd-reasoner *rules*))
-
-(defun run-test (game-name max-rule-size shuffle?)
+(defun run-test (game-name shuffle?)
   (let* ((scully.terms::*shuffle-variables* shuffle?)
-         (scully.gdl::*max-rule-size* max-rule-size)
          (gdl (scully.gdl::read-gdl (format nil "gdl/~(~A~)-grounded.gdl" game-name)))
-         (start (get-internal-real-time))
          (r (make-zdd-reasoner gdl))
-         (end (get-internal-real-time))
-         (elapsed (/ (- end start) internal-time-units-per-second))
          (sizes (reasoner-rule-tree-sizes r)))
-    (format t "~A ~D ~A ~D ~D ~,2F~%"
-            game-name
-            scully.gdl::*max-rule-size*
-            scully.terms::*shuffle-variables*
-            (length sizes)
-            (apply #'+ sizes)
-            elapsed)
-    (values)))
+    sizes))
+
+(defun percentile (sorted-numbers p)
+  (nth (truncate (* (/ p 100) (length sorted-numbers)))
+       sorted-numbers))
+
+(defun percentiles (numbers)
+  (let ((sorted (sort numbers #'<)))
+    (values (percentile sorted 5)
+            (percentile sorted 25)
+            (percentile sorted 50)
+            (percentile sorted 75)
+            (percentile sorted 95))))
 
-(defun test-all ()
-  (iterate (for game :in '(montyhall meier mastermind448 transit vis_pacman3p latenttictactoe stratego))
-           (run-test game 8 nil)
-           (iterate (repeat 10)
-                    (run-test game 8 t))))
+(defun run-shuffle-test (x game-name &optional (iterations 10))
+  (with-open-file (data "data-shuffling-rule-trees"
+                        :direction :output
+                        :if-exists :append
+                        :if-does-not-exist :create)
+    (let* ((unshuffled-size (apply #'+ (run-test game-name nil)))
+           (results (iterate
+                      (for i :from 1 :to iterations)
+                      (princ i) (princ #\space) (finish-output)
+                      (for shuffled-size = (apply #'+ (run-test game-name t)))
+                      (collect (/ shuffled-size unshuffled-size 1.0)))))
+      (fresh-line)
+      (multiple-value-bind (p5 p25 p50 p75 p95)
+          (percentiles results)
+        (format data "~D ~,5F ~,5F ~,5F ~,5F ~,5F 0.4 ~A~%"
+                x p5 p25 p50 p75 p95 game-name)))))
 
-;; (test-all)
+(defun run-shuffle-tests (iterations)
+  (iterate
+    (for game :in '(montyhall meier mastermind448 transit vis_pacman3p latenttictactoe stratego))
+    (pr game)
+    (for x :from 1)
+    (run-shuffle-test x game iterations)))
 
-;; (iterate
-;;   (repeat 10)
-;;   (run-test 'transit 12 nil))
-
+(run-shuffle-tests 50)