# Performs unit tests on networks and mutations

# UnitTest.tz (c) 2010 Jacob Schrum.
# See Simulation.tz for more copyright information

@use Abstract.

Abstract : Type1 {

}

Abstract : Type2 {

}

Abstract : UnitTest {
	+ variables:
		num-inputs (int).
		num-outputs (int).
		num-input-vectors (int).

	# Trivial archiving, since state consists of ints
	+ to archive:
		return 1.

	+ to dearchive:
		return 1.

	+ to init:
		num-inputs = 5.
		num-outputs = 2.
		num-input-vectors = 10.

	+ to run-tests:
		print "UNIT TESTS".
		print "--------------------------------------------".
		self test-subset.
		self test-set-equal.
		self test-compare.
		self test-compare-with-error.
		self test-angle-difference.
		self test-mode-counts.
		self test-mode-arbitrator.
		self test-mode-mutation.
		self test-nsga2.
		self test-nsga2-pref-point.
		self test-part-tug.
		self test-ctug-nsga2.
		self test-output-mode-merge.
		self test-output-mode-demotion.
		self test-output-node-removal. 
		self test-output-mode-deletion.
		self test-output-mode-deletion-with-output-recurrences.
		self test-crossover-with-multiple-modes.
		print "--------------------------------------------".

	+ to test message m (string) result r (int):
		if r : print "PASSED: $m".
		else print "---->FAILED: $m".

	+ to get-random-input-vectors num num-input-vectors (int) in num-inputs (int):
		input-vectors (list).
		i (int).
		j (int).

		input-vectors = {}.
		for i = 0, i < num-input-vectors, i++ :
		{
			push {} onto input-vectors.
			for j = 0, j < num-inputs, j++ :
			{
				push (random[ 2.0 ] - 1.0) onto (input-vectors{i}).
			}
		}

		return input-vectors.		

	+ to run-net-with-input-vectors network net (object) in input-vectors (list):
		i (int).
		output-vectors (list).

		net flush.
		output-vectors = {}.
		for i = 0, i < |input-vectors|, i++ :
		{
			net run-with inputs (input-vectors{i}).
			push (net get-all-outputs) onto output-vectors.
		}
		
		return output-vectors.

	+ to compare-output-sets set1 outputs1 (list) set2 outputs2 (list):
		i (int).
		j (int).

		if (|outputs1| != |outputs2|) : return 0.

		for i = 0, i < |outputs1|, i++ :
		{
			for j = 0, j < |outputs1{i}|, j++ :
			{
				if ((outputs1{i}{j}) != (outputs2{i}{j})) : return 0.
			}
		}

		return 1.

	+ to test-compare:
		outputs1 (list).
		outputs2 (list).

		outputs1 = {{1,2},{0.435,-0.234},{10.235,0.000002},{-0.213,-50.425}}.
		outputs2 = {{0.435,-0.234},{10.235,0.000002},{0,9},{-0.1568,23.679}}.

		self test message "Any output set is equal to itself" result (self compare-output-sets set1 outputs1 set2 outputs1).	
		self test message "Different output sets are not equal" result (!(self compare-output-sets set1 outputs1 set2 outputs2)).	

	+ to test-mode-arbitrator:
		net (object).
		input-vectors (list).
		output-vectors-net (list).
		output-vectors-arbitrator (list).
		i (int).
		mode-result (list).
		mode-arbitrator (object).

		net = new FreeNetwork.
		net random-initialize with-inputs num-inputs and-outputs num-outputs.

		input-vectors = (self get-random-input-vectors num num-input-vectors in num-inputs).
		output-vectors-net = (self run-net-with-input-vectors network net in input-vectors).

		mode-arbitrator = new ModeArbitrator.
		mode-arbitrator set-nodes-per-policy to num-outputs.
		mode-arbitrator set-non-deterministic to 0.
		mode-arbitrator set-use-preference-nodes to 1.

		output-vectors-arbitrator = {}.
		for i = 0, i < |output-vectors-net|, i++ :
		{
			mode-result = mode-arbitrator net-outputs-of outputs (output-vectors-net{i}).
			push (mode-result{ARBITRATE_OUTPUT}) onto output-vectors-arbitrator.
		}

		self test message "Compare raw net outputs to mode arbitrator outputs" 
		     result (self compare-output-sets set1 output-vectors-net set2 output-vectors-arbitrator).

	+ to test-output-mode-demotion:
		net (object).
		ga (object).
		mode-arbitrator (object).
		i (int).
		temp (int).
		modes-to-add (int).
		num-nodes (int).

		net = new FreeNetwork.
                net random-initialize with-inputs num-inputs and-outputs num-outputs.

		ga = new FreeNetGA.
		ga set-output-width to num-outputs.

		mode-arbitrator = new ModeArbitrator.
		mode-arbitrator set-nodes-per-policy to num-outputs.
		mode-arbitrator set-non-deterministic to 0.
		mode-arbitrator set-use-preference-nodes to 1.

		self test message "New network has only one mode" result (1 == (mode-arbitrator get-num-output-modes-from-net network net)).

		ga add-output-mode onto net.
		self test message "Network has two modes after one is added" 
			  result (2 == (mode-arbitrator get-num-output-modes-from-net network net)).

		num-nodes = (net get-number-of-nodes).

		ga demote-output-mode of net.

		self test message "Network has only one mode after demotion" result (1 == (mode-arbitrator get-num-output-modes-from-net network net)).
		self test message "Network has same number of nodes after demotion" result (num-nodes == (net get-number-of-nodes)).

		ga demote-output-mode of net.

		self test message "No mode demotion can happen when there is only one mode" 
			  result (1 == (mode-arbitrator get-num-output-modes-from-net network net)).

		modes-to-add = 5.
		for i = 0, i < modes-to-add, i++ :
		{
			ga add-output-mode onto net.
			temp = i + 1.
			self test message "Mode added to network" 
			          result ((1 + temp) == (mode-arbitrator get-num-output-modes-from-net network net)).
		}

		num-nodes = (net get-number-of-nodes).

		for i = i, i > 0, i-- :
		{
			ga demote-output-mode of net.
			temp = i - 1.
			self test message "One mode demoted away" 
			          result ((1 + temp) == (mode-arbitrator get-num-output-modes-from-net network net)).
			self test message "Same number of nodes after demotion" result (num-nodes == (net get-number-of-nodes)).
		}

	+ to test-output-mode-merge:
		net (object).
		ga (object).
		mode-arbitrator (object).
		i (int).
		temp (int).
		modes-to-add (int).

		net = new FreeNetwork.
                net random-initialize with-inputs num-inputs and-outputs num-outputs.

		ga = new FreeNetGA.
		ga set-output-width to num-outputs.

		mode-arbitrator = new ModeArbitrator.
		mode-arbitrator set-nodes-per-policy to num-outputs.
		mode-arbitrator set-non-deterministic to 0.
		mode-arbitrator set-use-preference-nodes to 1.

		self test message "New network has only one mode" result (1 == (mode-arbitrator get-num-output-modes-from-net network net)).

		ga add-output-mode onto net.
		self test message "Network has two modes after one is added" 
			  result (2 == (mode-arbitrator get-num-output-modes-from-net network net)).

		ga merge-output-modes of net.

		self test message "Network has only one mode after merge" result (1 == (mode-arbitrator get-num-output-modes-from-net network net)).

		ga merge-output-modes of net.

		self test message "No mode merge can happen when there is only one mode" 
			  result (1 == (mode-arbitrator get-num-output-modes-from-net network net)).

		modes-to-add = 5.
		for i = 0, i < modes-to-add, i++ :
		{
			ga add-output-mode onto net.
			temp = i + 1.
			self test message "Mode added to network" 
			          result ((1 + temp) == (mode-arbitrator get-num-output-modes-from-net network net)).
		}

		for i = i, i > 0, i-- :
		{
			ga merge-output-modes of net.
			temp = i - 1.
			self test message "One mode merged away" 
			          result ((1 + temp) == (mode-arbitrator get-num-output-modes-from-net network net)).
		}

	+ to test-mode-mutation:
		net (object).
		input-vectors (list).
		output-vectors-net (list).
		output-vectors-two-modes (list).
		output-vectors-mode1 (list).
		output-vectors-mode2 (list).
		output-vectors-arbitrator-mode1 (list).
		output-vectors-arbitrator-mode2 (list).
		i (int).
		mode-result (list).
		mode-arbitrator (object).
		ga (object).

		net = new FreeNetwork.
		net random-initialize with-inputs num-inputs and-outputs num-outputs.

		input-vectors = (self get-random-input-vectors num num-input-vectors in num-inputs).
		output-vectors-net = (self run-net-with-input-vectors network net in input-vectors).

		ga = new FreeNetGA.
		ga set-output-width to num-outputs.
		ga add-output-mode onto net.

		output-vectors-two-modes = (self run-net-with-input-vectors network net in input-vectors).

		output-vectors-mode1 = {}.
                output-vectors-mode2 = {}.
		for i = 0, i < |output-vectors-two-modes|, i++ :
		{
			push {} onto output-vectors-mode1.
			push (output-vectors-two-modes{i}{0}) onto (output-vectors-mode1{i}).
			push (output-vectors-two-modes{i}{1}) onto (output-vectors-mode1{i}).

			push {} onto output-vectors-mode2.
			push (output-vectors-two-modes{i}{3}) onto (output-vectors-mode2{i}).
			push (output-vectors-two-modes{i}{4}) onto (output-vectors-mode2{i}).
		}

		self test message "Compare outputs before mode mutation to outputs after mode mutation" 
		     result (self compare-output-sets set1 output-vectors-net set2 output-vectors-mode1).

		mode-arbitrator = new ModeArbitrator.
		mode-arbitrator set-nodes-per-policy to num-outputs.
		mode-arbitrator set-non-deterministic to 0.
		mode-arbitrator set-use-preference-nodes to 1.

		output-vectors-arbitrator-mode1 = {}.
		output-vectors-arbitrator-mode2 = {}.
		for i = 0, i < |output-vectors-two-modes|, i++ :
		{
			mode-result = mode-arbitrator mode-outputs-of outputs (output-vectors-two-modes{i}) mode 0.
			push mode-result onto output-vectors-arbitrator-mode1.
			mode-result = mode-arbitrator mode-outputs-of outputs (output-vectors-two-modes{i}) mode 1.
			push mode-result onto output-vectors-arbitrator-mode2.
		}

		self test message "Compare raw net outputs to arbitrator outputs after mode mutation" 
		     result ((self compare-output-sets set1 output-vectors-mode1 set2 output-vectors-arbitrator-mode1) &&
			     (self compare-output-sets set1 output-vectors-mode2 set2 output-vectors-arbitrator-mode2)).
	
	+ to compare-lists-allowing-error epsilon e (double) list1 l1 (list) list2 l2 (list):
		i (int).

		for i = 0, i < |l1|, i++ :
		{
			if (| l1{i} - l2{i} | > e) : return 0.
		}

		return 1.

	+ to test-compare-with-error:
		l1 (list).
		l2 (list).

		l1 = {1  ,2  ,3  ,4  ,5  ,6  ,7  }.
		l2 = {1.1,2.1,3.1,4.1,5.1,6.1,7.1}.		

		self test message "Comparisons with error allowances find the same values to be equal"
			  result (self compare-lists-allowing-error epsilon 0.0 list1 l1 list2 l1).
		
		self test message "Comparisons with error allowances find close values to be equal with appropriate error"
			  result (self compare-lists-allowing-error epsilon 0.11 list1 l1 list2 l2).

		self test message "Comparisons with error allowances find close values to be different with small error"
			  result (!(self compare-lists-allowing-error epsilon 0.01 list1 l1 list2 l2)).

	+ to test-angle-difference:
		math (object).
		vector-list1 (list).
		vector-list2 (list).
		answer-list (list).
		calculated-answers (list).
		i (int).

		vector-list1 = {(1,2,3),(0,1,0),(1,0,0) ,(0,0,0)}.
		vector-list2 = {(1,2,3),(5,0,0),(-1,0,0),(1,1,1)}.
		answer-list =  {0      ,(PI/2) ,PI      ,0      }.

		math = new Math3D.
		calculated-answers = {}.
		for i = 0, i < |answer-list|, i++ :
		{
			push (math signed-angle-difference between (vector-list1{i}) and (vector-list2{i})) onto calculated-answers.
		}

		self test message "Proper calculation of angles between vectors" 
		     result (self compare-lists-allowing-error epsilon VECTOR_EPSILON list1 answer-list list2 calculated-answers).

	+ to test-mode-counts:
		net (object).
		i (int).
		mode-arbitrator (object).
		ga (object).
		modes-to-add (int).
		temp (int).

		net = new FreeNetwork.
		net random-initialize with-inputs num-inputs and-outputs num-outputs.

		ga = new FreeNetGA.
		ga set-output-width to num-outputs.

		mode-arbitrator = new ModeArbitrator.
		mode-arbitrator set-nodes-per-policy to num-outputs.
		mode-arbitrator set-non-deterministic to 0.
		mode-arbitrator set-use-preference-nodes to 1.

		self test message "New network has only one mode" result (1 == (mode-arbitrator get-num-output-modes-from-net network net)).

		modes-to-add = 5.
		for i = 0, i < modes-to-add, i++ :
		{
			ga add-output-mode onto net.
			temp = i + 1.
			self test message "Correct network mode count after $temp mode(s) are added" 
			          result ((1 + temp) == (mode-arbitrator get-num-output-modes-from-net network net)).
		}

	+ to subset subset s1 (list) superset s2 (list):
		i (int).
		j (int).
		found (int).
		
		for i = 0, i < |s1|, i++ :
		{
			found = 0.
			for j = 0, j < |s2|, j++ :
			{
				if (s1{i} == s2{j}) : found = 1.	
			}
			if !found : return 0.
		}

		return 1.

	+ to test-subset:
		s1 (list).
		s2 (list).
		s3 (list).

		s1 = {1,2,3}.
		s2 = {1,2,3,4,5}.
		s3 = {1,2,4}.

		self test message "Correct result for when one set is subset of another" result (self subset subset s1 superset s2).
		
		self test message "Correct result for when one set is actually a superset of another" result (!(self subset subset s2 superset s1)).

		self test message "Correct result for when one set is neither subset nor superset of another" result (!(self subset subset s1 superset s3)).

	+ to set-equal set1 s1 (list) set2 s2 (list):
		return ((self subset subset s1 superset s2) && (self subset subset s2 superset s1)).

	+ to test-set-equal:
		s1 (list).
		s2 (list).
		s3 (list).
		s4 (list).

		s1 = {1,2,3}.
		s2 = {1,2,3,4,5}.
		s3 = {1,2,4}.
		s4 = {3,1,2}.
	
		self test message "Set is equal to itself" result (self set-equal set1 s1 set2 s1).

		self test message "Set is equal to identical set with different list order" result (self set-equal set1 s1 set2 s4).

		self test message "Set not equal to set of different size" result (!(self set-equal set1 s1 set2 s2)).

		self test message "Set not equal to set of same size" result (!(self set-equal set1 s1 set2 s3)).

	+ to test-nsga2:
		nsga2 (object).
		result (list).
		fitnesses (list).
		objectResult (object).

		nsga2 = new NSGA2.
		nsga2 set-last-front-selection to LAST_FRONT_CROWDING_DISTANCE.

		fitnesses = {{0,{0,0}},{1,{1,1}},{2,{1,0}},{3,{0,1}},{4,{1,2}},{5,{2,2}},{6,{2,1}},{7,{0,2}},{8,{2,0}},{9,{0,3}}}.
		result = (nsga2 get-best choose 2 of-population fitnesses).
		self test message "NSGA2 identifies first Pareto front correctly" result (self set-equal set1 result set2 {5,9}).

		fitnesses = {{0,{0,0}},{1,{1,1}},{2,{1,0}},{3,{0,1}},{4,{1,2}},{5,{2,2}},{6,{2,1}},{7,{0,2}},{8,{2,0}},{9,{0,3}}}.
		result = (nsga2 get-best choose 4 of-population fitnesses).
		self test message "NSGA2 identifies second Pareto front correctly" result (self set-equal set1 result set2 {5,9,4,6}).

		fitnesses = {{0,{0,0}},{1,{1,1}},{2,{1,0}},{3,{0,1}},{4,{1,2}},{5,{2,2}},{6,{2,1}},{7,{0,2}},{8,{2,0}},{9,{0,3}}}.
		result = (nsga2 get-best choose 7 of-population fitnesses).
		self test message "NSGA2 identifies third Pareto front correctly" result (self set-equal set1 result set2 {5,9,4,6,7,8,1}).

		fitnesses = {{0,{0,0}},{1,{1,1}},{2,{1,0}},{3,{0,1}},{4,{1,2}},{5,{2,2}},{6,{2,1}},{7,{0,2}},{8,{2,0}},{9,{0,3}}}.
		result = (nsga2 get-best choose 9 of-population fitnesses).
		self test message "NSGA2 identifies fourth Pareto front correctly" result (self set-equal set1 result set2 {3,2,5,9,4,6,7,8,1}).

		fitnesses = {{0,{0,0}},{1,{1,1}},{2,{1,0}},{3,{0,1}},{4,{1,2}},{5,{2,2}},{6,{2,1}},{7,{0,2}},{8,{2,0}},{9,{0,3}}}.
		result = (nsga2 get-best choose 10 of-population fitnesses).
		self test message "NSGA2 identifies fifth Pareto front correctly" result (self set-equal set1 result set2 {0,3,2,5,9,4,6,7,8,1}).

		objectResult = (nsga2 get-best between (new Type1) and (new Type2) given {0,{0},0,0} and {0,{0},0,1}).
		self test message "Binary tournament favors lower rank" result (objectResult is a "Type1").

		objectResult = (nsga2 get-best between (new Type1) and (new Type2) given {0,{0},0,1} and {0,{0},100000,1}).
		self test message "Binary tournament favors high crowding distance if rank is the same" result (objectResult is a "Type2").

		objectResult = (nsga2 get-best between (new Type1) and (new Type2) given {0,{0},0,0} and {0,{0},100000,1}).
		self test message "Binary tournament knows crowding distance is irrelevant if rank is lower" result (objectResult is a "Type1").

	+ to test-nsga2-pref-point:
		nsga2 (object).
		result (list).
		fitnesses (list).

		nsga2 = new NSGA2.
		nsga2 set-preferred-point to {0.1, 10}.
		nsga2 set-last-front-selection to LAST_FRONT_RANK_PREFERENCE_POINT.

		fitnesses = {{0,{10,8}},{1,{7,1}},{2,{11,2}},{3,{0.1,9}},{4,{3,4}}}.
		result = (nsga2 get-best choose 1 of-population fitnesses).
		self test message "PrefNSGA2 prefers point closest to preference point" result (self set-equal set1 result set2 {3}).

		fitnesses = {{0,{10,8}},{1,{7,1}},{2,{11,2}},{3,{0.1,9}},{4,{3,4}}}.
		result = (nsga2 get-best choose 2 of-population fitnesses).
		self test message "PrefNSGA2 prefers 2 points closest to preference point" result (self set-equal set1 result set2 {3,0}).

		fitnesses = {{0,{10,8}},{1,{7,1}},{2,{11,2}},{3,{0.1,9}},{4,{3,4}}}.
		result = (nsga2 get-best choose 3 of-population fitnesses).
		self test message "PrefNSGA2 prefers first Pareto front" result (self set-equal set1 result set2 {3,0,2}).

		fitnesses = {{0,{10,8}},{1,{7,1}},{2,{11,2}},{3,{0.1,9}},{4,{3,4}}}.
		result = (nsga2 get-best choose 4 of-population fitnesses).
		self test message "PrefNSGA2 prefers point in second front that is closer to pref point" result (self set-equal set1 result set2 {3,0,2,4}).

	+ to test-ctug-nsga2:
		nsga2 (object).
		result (list).
		fitnesses (list).

		nsga2 = new CTUG-NSGA2.
		nsga2 set-last-front-selection to LAST_FRONT_CROWDING_DISTANCE.
		nsga2 set-goal-constraints from { {0,0,0,0,0,0,0,0,0, 10, 0,1}, {0,0,0,0,0,0,0,0,0, 5, 0,1} }.


		fitnesses = {{0,{5,13}},{1,{4,6}},{2,{6,2}},{3,{12,11}},{4,{15,10}},{5,{16,8}},{6,{11,1}},{7,{20,3}},{8,{14,7}},{9,{13,9}}}.
		result = (nsga2 get-best choose 3 of-population fitnesses).
		self test message "CTUG-NSGA2 Prefers points in front that don't violate constraints" result (self set-equal set1 result set2 {3,4,5}).

		fitnesses = {{0,{5,13}},{1,{4,6}},{2,{6,2}},{3,{12,11}},{4,{15,10}},{5,{16,8}},{6,{11,1}},{7,{20,3}},{8,{14,7}},{9,{13,9}}}.
		result = (nsga2 get-best choose 5 of-population fitnesses).
		self test message "CTUG-NSGA2 Prefers points that don't violate constraints" result (self set-equal set1 result set2 {3,4,5,8,9}).

		fitnesses = {{0,{5,13}},{1,{4,6}},{2,{6,2}},{3,{12,11}},{4,{15,10}},{5,{16,8}},{6,{11,1}},{7,{20,3}},{8,{14,7}},{9,{13,9}}}.
		result = (nsga2 get-best choose 6 of-population fitnesses).
		self test message "CTUG-NSGA2 Settles for points that violate constraints the least" result (self set-equal set1 result set2 {3,4,5,8,9,7}).

		fitnesses = {{0,{5,13}},{1,{4,6}},{2,{6,2}},{3,{12,11}},{4,{15,10}},{5,{16,8}},{6,{11,1}},{7,{20,3}},{8,{14,7}},{9,{13,9}}}.
		result = (nsga2 get-best choose 9 of-population fitnesses).
		self test message "CTUG-NSGA2 points violating fewer constraints have smaller total violation" 
		     result (self set-equal set1 result set2 {3,4,5,8,9,7,6,0,1}).

	+ to test-part-tug:
		nsga2 (object).
		result (list).
		fitnesses (list).

		nsga2 = new PartTUG-NSGA2.
		nsga2 set-objective-goals to ( {10, 10} ).

		fitnesses = {{0,{5,13}},{1,{4,6}},{2,{6,2}},{3,{12,11}},{4,{15,11}},{5,{16,8}},{6,{11,1}},{7,{20,3}},{8,{14,7}},{9,{13,9}}}.
		result = (nsga2 get-best choose 1 of-population fitnesses).
		self test message "Partitioning TUG prefers points achieving all goals" result (self set-equal set1 result set2 {4}).

		fitnesses = {{0,{5,13}},{1,{4,6}},{2,{6,2}},{3,{12,11}},{4,{15,11}},{5,{16,8}},{6,{11,1}},{7,{20,3}},{8,{14,7}},{9,{13,9}}}.
		result = (nsga2 get-best choose 2 of-population fitnesses).
		self test message "Partitioning TUG prefers points achieving all goals" result (self set-equal set1 result set2 {4,3}).

		fitnesses = {{0,{5,13}},{1,{4,6}},{2,{6,2}},{3,{12,11}},{4,{15,11}},{5,{16,8}},{6,{11,1}},{7,{20,3}},{8,{14,7}},{9,{13,9}}}.
		result = (nsga2 get-best choose 4 of-population fitnesses).
		self test message "Partitioning TUG then selects from partitions achieving at least one goal" 
			result (self set-equal set1 result set2 {4,3,0,9}).

		fitnesses = {{0,{5,13}},{1,{4,6}},{2,{6,2}},{3,{12,11}},{4,{15,11}},{5,{16,8}},{6,{11,1}},{7,{20,3}},{8,{14,7}},{9,{13,9}}}.
		result = (nsga2 get-best choose 8 of-population fitnesses).
		self test message "All members of partitions at the same level are favored" 
			result (self set-equal set1 result set2 {4,3,0,9,5,8,7,6}).

		fitnesses = {{0,{5,13}},{1,{4,6}},{2,{6,2}},{3,{12,11}},{4,{15,11}},{5,{16,8}},{6,{11,1}},{7,{20,3}},{8,{14,7}},{9,{13,9}},{10,{1,1}}}.
		result = (nsga2 get-best choose 10 of-population fitnesses).
		self test message "In partition achieving no goals, favor Pareto front" 
			result (self set-equal set1 result set2 {4,3,0,9,5,8,7,6,1,2}).
		
		nsga2 set-objective-goals to ( {0, 0} ).
		fitnesses = {{0,{5,13}},{1,{4,6}},{2,{6,2}},{3,{12,11}},{4,{15,11}},{5,{16,8}},{6,{11,1}},{7,{20,3}},{8,{14,7}},{9,{13,9}},{10,{1,1}}}.
		result = (nsga2 get-best choose 4 of-population fitnesses).
		self test message "With everyone in best partition, favor Pareto front" 
			result (self set-equal set1 result set2 {0,4,5,7}).
	
		((controller get-commandline) set-bit-param named "empty-partition-support" to 1).

		nsga2 set-objective-goals to ( {10, 10} ).
		fitnesses = {{0,{5,13}},{1,{4,6}},{2,{6,2}},{3,{12,11}},{4,{15,11}},{5,{16,8}},{6,{11,1}},{7,{20,3}},{8,{14,7}},{9,{13,9}},{10,{1,1}}}.
		result = (nsga2 get-best choose 2 of-population fitnesses).
		self test message "Even when using partition support, favor partition achieving all goals" 
			result (self set-equal set1 result set2 {3,4}).
	
		fitnesses = {{0,{5,13}},{1,{4,6}},{2,{6,2}},{3,{12,11}},{4,{15,11}},{5,{16,8}},{6,{11,1}},{7,{20,3}},{8,{14,7}},{9,{13,9}},{10,{1,1}}}.
		result = (nsga2 get-best choose 4 of-population fitnesses).
		self test message "Even when using partition support, favor partitions at the same level" 
			result (self set-equal set1 result set2 {3,4,0,9}).
	
		fitnesses = {{0,{5,13}},{1,{4,6}},{2,{6,2}},{3,{12,11}},{4,{15,11}},{5,{16,8}},{6,{11,1}},{7,{20,3}},{8,{14,7}},{9,{13,9}},{10,{1,1}}}.
		result = (nsga2 get-best choose 8 of-population fitnesses).
		self test message "When partition empties, start pulling from support partition" 
			result (self set-equal set1 result set2 {3,4,0,9,1,2,5,8}).
	
		fitnesses = {{0,{5,13}},{1,{4,6}},{2,{6,2}},{3,{12,11}},{4,{15,11}},{5,{16,8}},{6,{11,1}},{7,{20,3}},{8,{14,7}},{9,{13,9}},{10,{1,1}}}.
		result = (nsga2 get-best choose 10 of-population fitnesses).
		self test message "Pull from support until there is none" 
			result (self set-equal set1 result set2 {3,4,0,9,1,2,5,8,10,7}).
	
		((controller get-commandline) set-bit-param named "empty-partition-support" to 0).
		((controller get-commandline) set-bit-param named "even-partition-selection" to 1).

		fitnesses = {{0,{5,13}},{1,{4,6}},{2,{6,2}},{3,{12,11}},{4,{15,11}},{5,{16,8}},{6,{11,1}},{7,{20,3}},{8,{14,7}},{9,{13,9}},{10,{1,1}}}.
		result = (nsga2 get-best choose 2 of-population fitnesses).
		self test message "Even partition selection still favors partition achieving all goals" 
			result (self set-equal set1 result set2 {3,4}).
			
		fitnesses = {{0,{5,13}},{1,{4,6}},{2,{6,2}},{3,{12,11}},{4,{15,11}},{5,{16,8}},{6,{11,1}},{7,{20,3}},{8,{14,7}},{9,{13,9}},{10,{9,9}}}.
		result = (nsga2 get-best choose 9 of-population fitnesses).
		self test message "When partitions empty out, algorithm selects evenly from the rest" 
			result (self set-equal set1 result set2 {3,4,0,9,10,1,2,5,8}).

	+ to test-output-node-removal:
		net (object).
		i (int).
		num-nodes (int).
		nodes-to-add (int).
		num-links (int).

		net = new FreeNetwork.
                net random-initialize with-inputs num-inputs and-outputs num-outputs.

		self test message "New network has $num-outputs outputs" result (num-outputs == (net get-number-outputs)).
		num-nodes = (net get-number-of-nodes).

		#net print-network.
		nodes-to-add = 5.
		for i = 0, i < nodes-to-add, i++ : {
			net splice-node-between-nodes at-position 0 and-position 0 with-innovation-number (20 + i).
			self test message "One node added in sequence" result ((num-nodes + i + 1) == (net get-number-of-nodes)).
			if ( (20 + i) % 2 == 0 ) : {
				net add-connection-between-nodes-with-innovation-numbers of (20 + i) and (20 + i) with-synaptic-weight 1.0.
			}
			#net print-network.
		}

		for i = 1, i < (net get-number-inputs), i++ : {
			net add-connection-between-nodes at-position i and (((net get-number-of-nodes) - (net get-number-outputs)) - 2) 
							 with-synaptic-weight 1.0.
			net splice-node-between-nodes at-position i and-position 2 with-innovation-number (40 + i).
			self test message "One node added in parallel" result ((num-nodes + nodes-to-add + i) == (net get-number-of-nodes)).
			if ( (40 + i) % 2 == 0 ) : {
				net add-connection-between-nodes-with-innovation-numbers of (40 + i) and (40 + i) with-synaptic-weight 1.0.
			}
			#net print-network.
		}

		num-links = (net get-number-of-links).
		net add-connection-between-nodes-with-innovation-numbers of (20) and (44) with-synaptic-weight 1.0.
		self test message "Added unusual recurrent link" result ((num-links + 1) == (net get-number-of-links)).

		#net print-network.

		net remove-output-node at-position ((net get-number-of-nodes) - (net get-number-outputs)).
		self test message "Output node removal cascades to remove intermediate hidden nodes" result ((num-nodes - 1) == (net get-number-of-nodes)).

		#print "old nodes: $num-nodes".
		#print "current:", (net get-number-of-nodes).
		#net print-network.

		self test message "Network now has one fewer output" result ((num-outputs - 1) == (net get-number-outputs)).

	+ to test-output-mode-deletion:
		net (object).
		ga (object).
		mode-arbitrator (object).
		i (int).
		j (int).
		k (int).
		temp (int).
		modes-to-add (int).
		input-vectors (list).
		output-vectors-net (list).
		reduced-output-vectors-net (list).

		net = new FreeNetwork.
                net random-initialize with-inputs num-inputs and-outputs num-outputs.

		ga = new FreeNetGA.
		ga set-output-width to num-outputs.
		ga set-multi-modal-random-source to 1.

		mode-arbitrator = new ModeArbitrator.
		mode-arbitrator set-nodes-per-policy to num-outputs.
		mode-arbitrator set-non-deterministic to 0.
		mode-arbitrator set-use-preference-nodes to 1.

		self test message "New network has only one mode" result (1 == (mode-arbitrator get-num-output-modes-from-net network net)).
		#net print-network.

		ga add-output-mode onto net comes-from 1.
		self test message "Network has two modes after one is added" 
			  result (2 == (mode-arbitrator get-num-output-modes-from-net network net)).
		#net print-network.

		ga delete-output-mode of net mode 0.

		self test message "Network has only one mode after deletion" result (1 == (mode-arbitrator get-num-output-modes-from-net network net)).
		#net print-network.

		ga delete-output-mode of net mode 0.

		self test message "No mode deletion can happen when there is only one mode" 
			  result (1 == (mode-arbitrator get-num-output-modes-from-net network net)).
		#net print-network.

		modes-to-add = 5.
		for i = 0, i < modes-to-add, i++ :
		{
			ga add-output-mode onto net comes-from 1.
			temp = i + 1.
			self test message "Mode added to network" 
			          result ((1 + temp) == (mode-arbitrator get-num-output-modes-from-net network net)).
			#net print-network.
		}

		input-vectors = (self get-random-input-vectors num num-input-vectors in num-inputs).
		output-vectors-net = (self run-net-with-input-vectors network net in input-vectors).
		reduced-output-vectors-net = (self run-net-with-input-vectors network net in input-vectors).

		self test message "Before any deletion, do output comparison" 
		     result (self compare-output-sets set1 output-vectors-net set2 reduced-output-vectors-net).

		for i = i, i > 0, i-- :
		{
			ga delete-output-mode of net mode 0.
			temp = i - 1.
			self test message "One mode deleted" 
			          result ((1 + temp) == (mode-arbitrator get-num-output-modes-from-net network net)).
			#net print-network.

			net flush.
			reduced-output-vectors-net = (self run-net-with-input-vectors network net in input-vectors).
			for j = 0, j < |output-vectors-net|, j++ : {
				for k = 0, k <= num-outputs, k++ : {
					remove (output-vectors-net{j}){0}.
				}
			}
			#print "|RED|=",|reduced-output-vectors-net{0}|,",|OUT|=",|output-vectors-net{0}|.
			#print "RED", reduced-output-vectors-net.
			#print "OUT", output-vectors-net.	
	
			self test message "Compare outputs before deletion (minus deleted output) with after deletion" 
			     result (self compare-output-sets set1 output-vectors-net set2 reduced-output-vectors-net).
		}

		#net print-network.

	+ to test-output-mode-deletion-with-output-recurrences:
		net (object).
		ga (object).
		mode-arbitrator (object).
		i (int).
		first-output (int).
		temp (int).
		modes-to-add (int).

		net = new FreeNetwork.
                net random-initialize with-inputs num-inputs and-outputs num-outputs.

		ga = new FreeNetGA.
		ga set-output-width to num-outputs.
		ga set-multi-modal-random-source to 1.

		mode-arbitrator = new ModeArbitrator.
		mode-arbitrator set-nodes-per-policy to num-outputs.
		mode-arbitrator set-non-deterministic to 0.
		mode-arbitrator set-use-preference-nodes to 1.

		first-output = (net get-number-of-nodes) - (net get-number-outputs).

		self test message "New network has only one mode" result (1 == (mode-arbitrator get-num-output-modes-from-net network net)).

		modes-to-add = 5.
		for i = 0, i < modes-to-add, i++ :
		{
			ga add-output-mode onto net comes-from 1.
			temp = i + 1.
			self test message "Mode added to network" 
			          result ((1 + temp) == (mode-arbitrator get-num-output-modes-from-net network net)).
			#net print-network.
		}

		#net print-network.

		for i = 0, i < modes-to-add, i++ :
		{
			net add-connection-between-nodes-with-innovation-numbers of (3 * (i + 1)) and 0 with-synaptic-weight 1.0.
		}

		self test message "Network closed before mode deletion" result (net is-closed).

		ga delete-output-mode of net mode 0.

		self test message "Network still closed after deleting recurrent target in output layer" result (net is-closed).

		#net print-network.

	+ to test-crossover-with-multiple-modes:
		net1 (object).
		net2 (object).
		netCross (object).
		ga (object).
		mode-arbitrator (object).
		
		mode-arbitrator = new ModeArbitrator.
		mode-arbitrator set-nodes-per-policy to num-outputs.
		mode-arbitrator set-non-deterministic to 0.
		mode-arbitrator set-use-preference-nodes to 1.

		net1 = new FreeNetwork.
                net1 random-initialize with-inputs num-inputs and-outputs num-outputs.

		net2 = new FreeNetwork.
                net2 random-initialize with-inputs num-inputs and-outputs num-outputs.

		ga = new FreeNetGA.
                ga set-output-width to num-outputs.

		ga add-output-mode onto net1.
		ga add-output-mode onto net2.
		
		self test message "Mode mutation gave net1 a new mode" result (2 == (mode-arbitrator get-num-output-modes-from-net network net1)).
		self test message "Mode mutation gave net2 a new mode" result (2 == (mode-arbitrator get-num-output-modes-from-net network net2)).

		((controller get-commandline) set-num-param named "crossover-rate" to 1.1).
		netCross = (ga crossover this-genome net1 with-this-genome net2). 
		
		self test message "Child gets a new mode from each, for a total of 3 modes" 
			  result (3 == (mode-arbitrator get-num-output-modes-from-net network netCross)).
}