Add randomness

1 files changed, 37 insertions, 31 deletions

diff --git a/evaluate.py b/evaluate.py
index ff52c30..2fb805d 100755
--- a/evaluate.py
+++ b/evaluate.py
@@ -11,6 +11,7 @@ import logging
 from logging import debug
 from sklearn.cluster import KMeans
 from sklearn.ensemble import RandomForestClassifier
+from sklearn.ensemble import ExtraTreesClassifier 
 from sklearn.ensemble import RandomTreesEmbedding
 from sklearn.metrics import accuracy_score
 import time 
@@ -25,6 +26,8 @@ parser.add_argument("-D", "--treedepth", help="depth of trees", type=int, defaul
 parser.add_argument("-v", "--verbose", help="Use verbose output", action='store_true')
 parser.add_argument("-t", "--timer", help="Display execution time", action='store_true')
 parser.add_argument("-T", "--testmode", help="Testmode", action='store_true')
+parser.add_argument("-E", "--embest", help="RandomTreesEmbedding estimators", type=int, default=256)
+parser.add_argument("-r", "--randomness", help="Randomness parameter", type=int, default=0)
 
 args = parser.parse_args()
 if args.verbose:
@@ -34,7 +37,7 @@ def make_histogram(data, model, args):
     if args.kmean:
         hist_size = args.kmean
     else:
-        hist_size = args.estimators*args.leaves
+        hist_size = args.embest*args.leaves
 
     histogram = np.zeros((data.shape[0], data.shape[1],hist_size)) 
     for i in range(data.shape[0]):
@@ -53,20 +56,22 @@ def run_model (data, train, test, train_part, args):
     
     if (args.kmean):
         logging.debug("Computing KMeans with", train_part.shape[0], "keywords")
-        kmeans = KMeans(n_clusters=args.kmean, n_init=args.estimators, random_state=0).fit(train_part)
+        kmeans = KMeans(n_clusters=args.kmean, n_init=1, random_state=0).fit(train_part)
         hist_train = make_histogram(train, kmeans, args)
         hist_test = make_histogram(test, kmeans, args)
     else:
-        trees = RandomTreesEmbedding(max_leaf_nodes=args.leaves, n_estimators=args.estimators, random_state=0).fit(train_part)
+        trees = RandomTreesEmbedding(max_leaf_nodes=args.leaves, n_estimators=args.embest, random_state=0).fit(train_part)
         hist_train = make_histogram(train, trees, args)
         hist_test = make_histogram(test, trees, args)
  
     logging.debug("Generating histograms")
     
-
     logging.debug("Keywords shape", hist_train.shape, "\n")
     logging.debug("Planting trees...")
-    clf = RandomForestClassifier(n_estimators=args.estimators, max_depth=args.treedepth, random_state=0)
+    if args.randomness:
+        clf = RandomForestClassifier(max_features=args.randomness, n_estimators=args.estimators, max_depth=args.treedepth, random_state=0)
+    else:
+        clf = RandomForestClassifier(n_estimators=args.estimators, max_depth=args.treedepth, random_state=0)
     clf.fit(
             hist_train.reshape((hist_train.shape[0]*hist_train.shape[1], hist_train.shape[2])),
             np.repeat(np.arange(hist_train.shape[0]), hist_train.shape[1]))
@@ -75,6 +80,8 @@ def run_model (data, train, test, train_part, args):
 
     test_pred = clf.predict(hist_test.reshape((hist_test.shape[0]*hist_test.shape[1], hist_test.shape[2])))
     test_label = np.repeat(np.arange(hist_test.shape[0]), hist_test.shape[1])
+    train_pred = clf.predict(hist_train.reshape((hist_train.shape[0]*hist_train.shape[1], hist_train.shape[2])))
+    train_label = np.repeat(np.arange(hist_train.shape[0]), hist_train.shape[1])
 
     if args.timer:
         end = time.time()
@@ -83,7 +90,7 @@ def run_model (data, train, test, train_part, args):
         skplt.metrics.plot_confusion_matrix(test_pred, test_label, normalize=True)
         plt.show()
 
-    return accuracy_score(test_pred, test_label)
+    return accuracy_score(test_pred, test_label)#, accuracy_score(train_pred, train_label), end-start
  
 
 
@@ -95,35 +102,34 @@ def main():
     logging.debug("Verbose is on")
 
     if args.testmode:
-        cnt = 0
-        acc = np.zeros((5,5))
-        for i in range(5):
-            args.estimators = (i+1)*200
-            cnt+=1
-            for j in range(5):
-                args.treedepth = j + 1
-                print("Step ", cnt)
-                acc[i][j] = run_model (data, train, test, train_part, args) 
-                print("Accuracy ",acc[i][j])
-                cnt+=1
-        fig, ax = plt.subplots()
-        im = ax.imshow(acc)
-        ax.set_xticks(np.arange(5))
-        ax.set_yticks(np.arange(5))
-        ax.set_xlabel('Number of trees')
-        ax.set_ylabel('Tree depth')
+        acc = np.zeros((3,100))
+        a = np.zeros(100)
+        for i in range(100):
+            if i <= 10:
+                args.kmean = (i+1) 
+            else:
+                args.kmean = 15*i
+            a[i] = args.kmean
+            print("Kmeans: ",args.kmean)
+            acc[0][i], acc[1][i], acc[2][i] = run_model (data, train, test, train_part, args) 
+            print("Accuracy test:",acc[0][i], "Accuracy train:", acc[1][i])
+
+        plt.plot(a,1-acc[0]) 
+        plt.plot(a,1-acc[1]) 
+        plt.ylabel('Normalized Classification Error')
+        plt.xlabel('Vocabulary size')
+        plt.title('Classification error varying vocabulary size')
+        plt.legend(('Test','Train'),loc='upper right')
+        plt.show()
+        plt.plot(a,acc[2]) 
+        plt.ylabel('Time (s)')
+        plt.xlabel('Vocabulary size')
+        plt.title('Time complexity varying vocabulary size')
+        plt.show()
 
-        # Loop over data dimensions and create text annotations.
-        for i in range(5):
-                for j in range(5):
-                            text = ax.text(j, i, acc[i, j], ha="center", va="center", color="w")
-                            ax.set_title("Accuracy varying hyper-parameters")
-                            fig.tight_layout()
-                            plt.show()
     else:
         acc = run_model (data, train, test, train_part, args)
         print(acc)
     
 if __name__ == "__main__":
     main()
-