_initialLayerRepairTest.py

from keras.layers import Conv3D, MaxPooling3D
from keras.models import Sequential
from keras import optimizers
from keras import regularizers
from keras import initializers
from keras.regularizers import l2
from keras.layers import TimeDistributed, Flatten, LSTM, Dense, Activation, ZeroPadding2D, Dropout, BatchNormalization, RepeatVector, Bidirectional
from keras.layers import Conv2D, MaxPooling2D, AveragePooling2D, GlobalAveragePooling2D
from keras.layers import MaxPool2D, Input
from keras import backend as K
from tensorflow.keras.optimizers import Adam

# for later versions:
#from tensorflow.keras.optimizers import Adam
from keras.optimizers import adam_v2
import tensorflow as tf

import sys
import os
import json
import random
import numpy as np
from copy import deepcopy

b =  {"CONV":["CONV_32_3x3_KR00001",
      "CONV_32_5x5_KR00001",
      "CONV_64_3x3_KR00001",
      "CONV_64_5x5_KR00001",
      "CONV_128_3x3_KR00001",
      "CONV_128_5x5_KR00001",
      "CONV_32_3x3_NOL2",
      "CONV_32_5x5_NOL2",
      "CONV_64_3x3_NOL2",
      "CONV_64_5x5_NOL2",
      "CONV_128_3x3_NOL2",
      "CONV_128_5x5_NOL2",
      "CONV_32_3x3_KR00001_TANH",
      "CONV_32_5x5_KR00001_TANH",
      "CONV_64_3x3_KR00001_TANH",
      "CONV_64_5x5_KR00001_TANH",
      "CONV_128_3x3_KR00001_TANH",
      "CONV_128_5x5_KR00001_TANH",
      "CONV_32_3x3_NOL2_TANH",
      "CONV_32_5x5_NOL2_TANH",
      "CONV_64_3x3_NOL2_TANH",
      "CONV_64_5x5_NOL2_TANH",
      "CONV_128_3x3_NOL2_TANH",
      "CONV_128_5x5_NOL2_TANH",
      "CONV_32_3x3_KR01",
      "CONV_32_5x5_KR01",
      "CONV_64_3x3_KR01",
      "CONV_64_5x5_KR01",
      "CONV_128_3x3_KR01",
      "CONV_128_5x5_KR01",
      "CONV_32_3x3_KR01_TANH",
      "CONV_32_5x5_KR01_TANH",
      "CONV_64_3x3_KR01_TANH",
      "CONV_64_5x5_KR01_TANH",
      "CONV_128_3x3_KR01_TANH",
      "CONV_128_5x5_KR01_TANH"
     ]}
a = {"GENOTYPE-PHENOTYPE MAPPING":
  {
    "INITIAL_CONV_32_3x3_KR00001": "Conv2D(32, (3, 3), kernel_regularizer=l2(0.0001), activation='relu', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_32_5x5_KR00001": "Conv2D(32, (5, 5), kernel_regularizer=l2(0.0001), activation='relu', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_64_3x3_KR00001": "Conv2D(64, (3, 3), kernel_regularizer=l2(0.0001), activation='relu', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_64_5x5_KR00001": "Conv2D(64, (5, 5), kernel_regularizer=l2(0.0001), activation='relu', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_128_3x3_KR00001": "Conv2D(128, (3, 3), kernel_regularizer=l2(0.0001), activation='relu', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_128_5x5_KR00001": "Conv2D(128, (5, 5), kernel_regularizer=l2(0.0001), activation='relu', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_32_3x3_NOL2": "Conv2D(32, (3, 3), activation='relu', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_32_5x5_NOL2": "Conv2D(32, (5, 5), activation='relu', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_64_3x3_NOL2": "Conv2D(64, (3, 3), activation='relu', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_64_5x5_NOL2": "Conv2D(64, (5, 5), activation='relu', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_128_3x3_NOL2": "Conv2D(128, (3, 3), activation='relu', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_128_5x5_NOL2": "Conv2D(128, (5, 5), activation='relu', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_32_3x3_KR00001_TANH": "Conv2D(32, (3, 3), kernel_regularizer=l2(0.0001), activation='tanh', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_32_5x5_KR00001_TANH": "Conv2D(32, (5, 5), kernel_regularizer=l2(0.0001), activation='tanh', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_64_3x3_KR00001_TANH": "Conv2D(64, (3, 3), kernel_regularizer=l2(0.0001), activation='tanh', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_64_5x5_KR00001_TANH": "Conv2D(64, (5, 5), kernel_regularizer=l2(0.0001), activation='tanh', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_128_3x3_KR00001_TANH": "Conv2D(128, (3, 3), kernel_regularizer=l2(0.0001), activation='tanh', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_128_5x5_KR00001_TANH": "Conv2D(128, (5, 5), kernel_regularizer=l2(0.0001), activation='tanh', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_32_3x3_NOL2_TANH": "Conv2D(32, (3, 3), activation='tanh', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_32_5x5_NOL2_TANH": "Conv2D(32, (5, 5), activation='tanh', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_64_3x3_NOL2_TANH": "Conv2D(64, (3, 3), activation='tanh', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_64_5x5_NOL2_TANH": "Conv2D(64, (5, 5), activation='tanh', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_128_3x3_NOL2_TANH": "Conv2D(128, (3, 3), activation='tanh', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_128_5x5_NOL2_TANH": "Conv2D(128, (5, 5), activation='tanh', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_32_3x3_KR01": "Conv2D(32, (3, 3), kernel_regularizer=l2(0.1), activation='relu', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_32_5x5_KR01": "Conv2D(32, (5, 5), kernel_regularizer=l2(0.1), activation='relu', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_64_3x3_KR01": "Conv2D(64, (3, 3), kernel_regularizer=l2(0.1), activation='relu', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_64_5x5_KR01": "Conv2D(64, (5, 5), kernel_regularizer=l2(0.1), activation='relu', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_128_3x3_KR01": "Conv2D(128, (3, 3), kernel_regularizer=l2(0.1), activation='relu', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_128_5x5_KR01": "Conv2D(128, (5, 5), kernel_regularizer=l2(0.1), activation='relu', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_32_3x3_KR01_TANH": "Conv2D(32, (3, 3), kernel_regularizer=l2(0.1), activation='tanh', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_32_5x5_KR01_TANH": "Conv2D(32, (5, 5), kernel_regularizer=l2(0.1), activation='tanh', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_64_3x3_KR01_TANH": "Conv2D(64, (3, 3), kernel_regularizer=l2(0.1), activation='tanh', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_64_5x5_KR01_TANH": "Conv2D(64, (5, 5), kernel_regularizer=l2(0.1), activation='tanh', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_128_3x3_KR01_TANH": "Conv2D(128, (3, 3), kernel_regularizer=l2(0.1), activation='tanh', kernel_initializer=initializers.he_uniform(seed=",
    "INITIAL_CONV_128_5x5_KR01_TANH": "Conv2D(128, (5, 5), kernel_regularizer=l2(0.1), activation='tanh', kernel_initializer=initializers.he_uniform(seed=",
    "CONV_32_3x3_KR00001": "Conv2D(32, (3, 3), kernel_regularizer=l2(0.0001), activation='relu')",
    "CONV_32_5x5_KR00001": "Conv2D(32, (5, 5), kernel_regularizer=l2(0.0001), activation='relu')",
    "CONV_64_3x3_KR00001": "Conv2D(64, (3, 3), kernel_regularizer=l2(0.0001), activation='relu')",
    "CONV_64_5x5_KR00001": "Conv2D(64, (5, 5), kernel_regularizer=l2(0.0001), activation='relu')",
    "CONV_128_3x3_KR00001": "Conv2D(128, (3, 3), kernel_regularizer=l2(0.0001), activation='relu')",
    "CONV_128_5x5_KR00001": "Conv2D(128, (5, 5), kernel_regularizer=l2(0.0001), activation='relu')",
    "CONV_32_3x3_NOL2": "Conv2D(32, (3, 3), activation='relu')",
    "CONV_32_5x5_NOL2": "Conv2D(32, (5, 5), activation='relu')",
    "CONV_64_3x3_NOL2": "Conv2D(64, (3, 3), activation='relu')",
    "CONV_64_5x5_NOL2": "Conv2D(64, (5, 5), activation='relu')",
    "CONV_128_3x3_NOL2": "Conv2D(128, (3, 3), activation='relu')",
    "CONV_128_5x5_NOL2": "Conv2D(128, (5, 5), activation='relu')",
    "CONV_32_3x3_KR00001_TANH": "Conv2D(32, (3, 3), kernel_regularizer=l2(0.0001), activation='tanh')",
    "CONV_32_5x5_KR00001_TANH": "Conv2D(32, (5, 5), kernel_regularizer=l2(0.0001), activation='tanh')",
    "CONV_64_3x3_KR00001_TANH": "Conv2D(64, (3, 3), kernel_regularizer=l2(0.0001), activation='tanh')",
    "CONV_64_5x5_KR00001_TANH": "Conv2D(64, (5, 5), kernel_regularizer=l2(0.0001), activation='tanh')",
    "CONV_128_3x3_KR00001_TANH": "Conv2D(128, (3, 3), kernel_regularizer=l2(0.0001), activation='tanh')",
    "CONV_128_5x5_KR00001_TANH": "Conv2D(128, (5, 5), kernel_regularizer=l2(0.0001), activation='tanh')",
    "CONV_32_3x3_NOL2_TANH": "Conv2D(32, (3, 3), activation='tanh')",
    "CONV_32_5x5_NOL2_TANH": "Conv2D(32, (5, 5), activation='tanh')",
    "CONV_64_3x3_NOL2_TANH": "Conv2D(64, (3, 3), activation='tanh')",
    "CONV_64_5x5_NOL2_TANH": "Conv2D(64, (5, 5), activation='tanh')",
    "CONV_128_3x3_NOL2_TANH": "Conv2D(128, (3, 3), activation='tanh')",
    "CONV_128_5x5_NOL2_TANH": "Conv2D(128, (5, 5), activation='tanh')",
    "CONV_32_3x3_KR01": "Conv2D(32, (3, 3), kernel_regularizer=l2(0.1), activation='relu')",
    "CONV_32_5x5_KR01": "Conv2D(32, (5, 5), kernel_regularizer=l2(0.1), activation='relu')",
    "CONV_64_3x3_KR01": "Conv2D(64, (3, 3), kernel_regularizer=l2(0.1), activation='relu')",
    "CONV_64_5x5_KR01": "Conv2D(64, (5, 5), kernel_regularizer=l2(0.1), activation='relu')",
    "CONV_128_3x3_KR01": "Conv2D(128, (3, 3), kernel_regularizer=l2(0.1), activation='relu')",
    "CONV_128_5x5_KR01": "Conv2D(128, (5, 5), kernel_regularizer=l2(0.1), activation='relu')",
    "CONV_32_3x3_KR01_TANH": "Conv2D(32, (3, 3), kernel_regularizer=l2(0.1), activation='tanh')",
    "CONV_32_5x5_KR01_TANH": "Conv2D(32, (5, 5), kernel_regularizer=l2(0.1), activation='tanh')",
    "CONV_64_3x3_KR01_TANH": "Conv2D(64, (3, 3), kernel_regularizer=l2(0.1), activation='tanh')",
    "CONV_64_5x5_KR01_TANH": "Conv2D(64, (5, 5), kernel_regularizer=l2(0.1), activation='tanh')",
    "CONV_128_3x3_KR01_TANH": "Conv2D(128, (3, 3), kernel_regularizer=l2(0.1), activation='tanh')",
    "CONV_128_5x5_KR01_TANH": "Conv2D(128, (5, 5), kernel_regularizer=l2(0.1), activation='tanh')",
    "BATCH_NORM_99": "BatchNormalization(momentum = 0.99)",
    "BATCH_NORM_9": "BatchNormalization(momentum = 0.9)",
    "BATCH_NORM_75": "BatchNormalization(momentum = 0.75)",
    "BATCH_NORM_6": "BatchNormalization(momentum = 0.6)",
    "BATCH_NORM_45": "BatchNormalization(momentum = 0.45)",
    "BATCH_NORM_3": "BatchNormalization(momentum = 0.3)",
    "BATCH_NORM_1": "BatchNormalization(momentum = 0.1)",
    "BATCH_NORM_01": "BatchNormalization(momentum = 0.01)",
    "AVGPOOL_2x2": "AveragePooling2D(pool_size=(2, 2),strides=2)",
    "MAXPOOL_2x2": "MaxPooling2D(pool_size=(2, 2),strides=2)",
    "AVGPOOL_3x3": "AveragePooling2D(pool_size=(3, 3),strides=3)",
    "MAXPOOL_3x3": "MaxPooling2D(pool_size=(3, 3),strides=3)",
    "AVGPOOL_5x5": "AveragePooling2D(pool_size=(5, 5),strides=5)",
    "MAXPOOL_5x5": "MaxPooling2D(pool_size=(5, 5),strides=5)",
    "DROPOUT_0.1_SEED0": "Dropout(0.1, seed=0)",
    "DROPOUT_0.2_SEED0": "Dropout(0.2, seed=0)",
    "DROPOUT_0.3_SEED0": "Dropout(0.3, seed=0)",
    "DROPOUT_0.4_SEED0": "Dropout(0.4, seed=0)",
    "DROPOUT_0.5_SEED0": "Dropout(0.5, seed=0)"
  }}

class GP_tree:
    def __init__(self):
        self.arity = 2
    
    def simplify(self, ind):
        startflag = True
        headNode = 'r0'
        simpleList = []
        if self.arity == 1:
            for i in reversed(range(len(ind))):  
                if(ind[i][1] == headNode and len(ind[i]) < 3 and startflag == True):
                    pass
                elif(ind[i][1] == headNode):
                    simpleList.append(ind[i])
                    headNode = str(ind[i][2])
                    startflag = False
        elif self.arity == 2:
            print("got here")
            ind2 = ind[::-1]
            simpleList = list(self.recursiveTraversal(ind2, headNode))

        # reverse before output
        return simpleList[::-1]

    def recursiveTraversal(self, ind, headNode):
        if not ind:
            return
          
        first, *rest = ind
    
        if first[1] == headNode and len(first) < 4:
            yield first
            headNode = first[2]         
        elif first[1] == headNode and len(first) == 4:
            yield first
            for i in range(2, 4):
                yield from self.recursiveTraversal(rest, headNode)
                headNode = first[i]

        yield from self.recursiveTraversal(rest, headNode)

def repairInitialLayer(indiv, repair_gene):
        maxRegister = 6
        startflag = True
        headNode = 'r0'
        effectiveList = []
        # Go backwards through program
        for i in reversed(range(len(indiv))):
            # what we want is '!= arity greater or equal 2'
            if(indiv[i][1] == headNode and len(indiv[i]) < 3 and startflag == True):
                pass
            elif(indiv[i][1] == headNode):
                effectiveList.append(i)
                headNode = str(indiv[i][2])
                startflag = False

        #print(effectiveList)
        #effectiveList.sort(reverse=True)
        #print(effectiveList)
        #print(len(effectiveList))
        if len(effectiveList) > 0:
            # Current effective indiv at list at 0 will become our nextReg
            nextRegIndex = effectiveList[-1]

            regIdx = random.randint(0, maxRegister-1)
            randomReg = 'r%d' % regIdx

            # Insert randomly before first line of effective code
            if nextRegIndex >=1:
                insertLocation = random.randint(0, nextRegIndex-1)
            else:
                insertLocation = 0
            indiv.insert(insertLocation, [repair_gene, indiv[nextRegIndex][2], randomReg])
        elif len(effectiveList) == 0:
            # If there is no effective code we will add single layer and asign our output reg
            regIdx = random.randint(0, maxRegister-1)
            randomReg = 'r%d' % regIdx

            # Insert randomly before first line of effective code
            if nextRegIndex >=1:
                insertLocation = random.randint(0, nextRegIndex-1)
            else:
                insertLocation = 0
            indiv.insert(insertLocation, [repair_gene, 'r0', randomReg])
        return indiv

def repairByRemoveLayer(indiv):
    startflag = True
    headNode = 'r0'
    effectiveList = []
    # Go backwards through program
    for i in reversed(range(len(indiv))):
        # what we want is '!= arity greater or equal 2'
        if(indiv[i][1] == headNode and len(indiv[i]) < 3 and startflag == True):
            pass
        elif(indiv[i][1] == headNode):
            effectiveList.append(i)
            headNode = str(indiv[i][2])
            startflag = False

    effectiveList.sort(reverse=True)
    if len(effectiveList) > 0:
        # Current effective indiv at list at 0 will become our nextReg
        del indiv[-1]
        # New last position
        indiv[-1][1] = 'r0'

    return indiv


def generateModel(ind_simplified, ind, seed, a, b, gp_tree):

        #if kwargs["num_obj"] == 1:
        K.clear_session()

        #os.environ['TF_GPU_ALLOCATOR'] = 'cuda_malloc_async'
        
        #gpus = tf.config.experimental.list_physical_devices('GPU')
        #for gpu in gpus:
        #    tf.config.experimental.set_memory_growth(gpu, True)
        # This should clear previous models from memory
        
        #print(ind)

        count = len(ind_simplified)

        insertion_flag = 0 # This means an convolutional insertion at the start was required
        mutated_zero_flag = 0 # This means that the the effective code was mutated out of the genotype (should be rare)
        remove_layer_count = 0 # Counts
        ran_out_count = 0 # This means the repair re-occurred until size zero, have mave a count just to be safe
        flattened_flag = 0
        flat_flag_record = 0
        #print(count)
        while count >= 0:
            try:
                model = Sequential()
                if len(ind_simplified) == 0:
                    # This should not happen often or at all, do we want our croosover and mutation to result in size zero?
                    # Need to see how this happens and record
                    print("WARNING: Individual of size zero created through mutation")
                    ind = gp_tree.newIndividual()
                    ind_simplified = gp_tree.simplify(ind)
                    count = len(ind_simplified)
                    if ran_out_count == 0:
                        mutated_zero_flag = 1 # Only count if this hasn't occurred as a result of the repair
                for i in range(len(ind_simplified)):
                    #print(ind_simplified[i][0])
                    if i == 0:
                        repair_list = b["CONV"]
                        if str(ind_simplified[i][0]) in repair_list:
                            initial_repair_gene = "INITIAL_" + str(ind_simplified[i][0])
                            str_start = a["GENOTYPE-PHENOTYPE MAPPING"][initial_repair_gene]
                            full_string = str_start + str(seed) + '), input_shape=' + str((64, 64, 3)) + ')'
                            model.add(eval(full_string))
                        else:
                            insertion_flag = 1
                            # We need a default layer to add if not convolutional - manditory repair
                            #TODO update to randomly select a convolutional layer
                            #random_number = random.random()
                            #print(b)
                            repair_gene = random.choice(repair_list)
                            initial_repair_gene = "INITIAL_" + str(repair_gene)
                            #print(ind_simplified)
                            ind = gp_tree.repairInitialLayer(ind, repair_gene)
                            ind_simplified = gp_tree.simplify(ind)
                            #print(ind_simplified)
                            #sys.exit()
                            count = len(ind_simplified)
                            str_start = a["GENOTYPE-PHENOTYPE MAPPING"][initial_repair_gene]
                            full_string = str_start + str(seed) + '), input_shape=' + str((64, 64, 3)) + ')'
                            #print(full_string)
                            model.add(eval(full_string))
                    else:
                        model.add(eval(a["GENOTYPE-PHENOTYPE MAPPING"][str(ind_simplified[i][0])]))
                    count = 1

                #model.summary()
                model.add(Flatten())
                
                # calculate number of classes
                flattened_size = model.layers[-1].output_shape[1]
                if flattened_size < 25000:
                    model.add(Dropout(0.2, seed=0))
                    model.add(Dense(512, activation='relu'))
                elif flattened_size < 50000:
                    model.add(Dropout(0.2, seed=0))
                    model.add(Dense(256, activation='relu'))
                elif flattened_size < 100000:
                    model.add(Dropout(0.2, seed=0))
                    model.add(Dense(128, activation='relu'))
                elif flattened_size > 1500000:
                    flattened_flag = 1
                    raise Exception('Flattened layer exceeds allowable number of nodes')
                    
                model.add(Dropout(0.2, seed=0))
                model.add(Dense(2, activation='softmax'))
                #print("getting here 3")
                model.summary()
                #optimizer = Adam(lr=0.001)

                # FS: need to check if additional metrics required for multi-objective compile
                
                model.compile(loss='categorical_crossentropy', 
                            metrics=['accuracy'],
                            optimizer='adam')
                break
            except Exception as e:
                #del model
                if flattened_flag == 1:
                    ind = gp_tree.newIndividual()
                    ind_simplified = gp_tree.simplify(ind)
                    #print(ind)
                    #print(ind_simplified)
                    #count = len(ind_simplified)
                    flattened_flag = 0
                    flat_flag_record = 1
                #count -= 1
                remove_layer_count += 1 # This can grow larger than the total number layers

                ind = repairByRemoveLayer(ind)
                ind_simplified = gp_tree.simplify(ind)
                count = len(ind_simplified)
                if count == 0:
                    print("Ran out")
                    ran_out_count += 1 # can run out multiple times but this should be rare
                    print(ind_simplified)
                    print(e)
            else:
                sys.exit(0)


            

        repair_stats = [insertion_flag,mutated_zero_flag,remove_layer_count,ran_out_count, flat_flag_record]
        #model.summary()
        return model, ind_simplified, ind, repair_stats

def main():
    indiv_test = [
    ['CONV_128_5x5_KR01_TANH', 'r4', 'r4'], 
    ['DROPOUT_0.2_SEED0', 'r1', 'r4'], 
    ['MAXPOOL_5x5', 'r1', 'r4'], 
    ['BATCH_NORM_1', 'r0', 'r1'], 
    ['AVGPOOL_2x2', 'r3', 'r2'], 
    ['DROPOUT_0.1_SEED0', 'r3', 'r0'], 
    ['CONV_64_3x3_NOL2', 'r1', 'r0'], 
    ['CONV_128_5x5_KR00001', 'r5', 'r3'], 
    ['DROPOUT_0.3_SEED0', 'r4', 'r1'], 
    ['AVGPOOL_3x3', 'r1', 'r1'], 
    ['CONV_64_5x5_KR00001', 'r4', 'r2'], 
    ['CONV_128_3x3_KR01_TANH', 'r4', 'r1'], 
    ['CONV_64_3x3_KR00001', 'r1', 'r5'], 
    ['BATCH_NORM_75', 'r1', 'r5'], 
    ['DROPOUT_0.5_SEED0', 'r1', 'r5'], 
    ['BATCH_NORM_01', 'r3', 'r2'], 
    ['CONV_64_3x3_KR01', 'r5', 'r5'], 
    ['CONV_64_3x3_KR01', 'r2', 'r1'], 
    ['CONV_64_5x5_KR00001_TANH', 'r0', 'r5'], 
    ['AVGPOOL_5x5', 'r4', 'r2'], 
    ['CONV_128_5x5_KR01', 'r3', 'r0'], 
    ['CONV_32_3x3_KR00001_TANH', 'r1', 'r1'], 
    ['BATCH_NORM_1', 'r0', 'r2'],
    ['CONV_128_5x5_KR01_TANH', 'r4', 'r0'], 
    ['DROPOUT_0.2_SEED0', 'r1', 'r4'], 
    ['MAXPOOL_5x5', 'r1', 'r4'], 
    ['BATCH_NORM_1', 'r0', 'r1'], 
    ['AVGPOOL_2x2', 'r3', 'r2'], 
    ['DROPOUT_0.1_SEED0', 'r3', 'r0'], 
    ['CONV_64_3x3_NOL2', 'r1', 'r0'], 
    ['CONV_128_5x5_KR00001', 'r5', 'r3'], 
    ['DROPOUT_0.3_SEED0', 'r4', 'r1'], 
    ['AVGPOOL_3x3', 'r1', 'r1'], 
    ['CONV_64_5x5_KR00001', 'r4', 'r2'], 
    ['CONV_128_3x3_KR01_TANH', 'r4', 'r1'], 
    ['CONV_64_3x3_KR00001', 'r1', 'r5'], 
    ['BATCH_NORM_75', 'r1', 'r5'], 
    ['DROPOUT_0.5_SEED0', 'r1', 'r5'], 
    ['BATCH_NORM_01', 'r3', 'r2'], 
    ['CONV_64_3x3_KR01', 'r5', 'r5'], 
    ['CONV_64_3x3_KR01', 'r2', 'r1'], 
    ['CONV_64_5x5_KR00001_TANH', 'r0', 'r5'], 
    ['AVGPOOL_5x5', 'r4', 'r2'], 
    ['CONV_128_5x5_KR01', 'r3', 'r0'], 
    ['CONV_32_3x3_KR00001_TANH', 'r1', 'r1'], 
    ['BATCH_NORM_1', 'r0', 'r2'],
    ]

    
    indiv_test = [
    ['CONV_128_5x5_KR01_TANH', 'r4', 'r4'], 
    ['DROPOUT_0.2_SEED0', 'r1', 'r4'], 
    ['MAXPOOL_5x5', 'r1', 'r4'], 
    ['BATCH_NORM_1', 'r0', 'r1'], 
    ['AVGPOOL_2x2', 'r3', 'r2'], 
    ['DROPOUT_0.1_SEED0', 'r3', 'r0'], 
    ['CONV_64_3x3_NOL2', 'r1', 'r0'], 
    ['BATCH_NORM_1', 'r0', 'r2'],
    ['CONV_128_5x5_KR01_TANH', 'r4', 'r0'], 
    ['DROPOUT_0.2_SEED0', 'r1', 'r4'], 
    ['MAXPOOL_5x5', 'r1', 'r4'], 
    ['BATCH_NORM_1', 'r0', 'r1'], 
    ['AVGPOOL_2x2', 'r3', 'r2'], 
    ['DROPOUT_0.1_SEED0', 'r3', 'r0'], 
    ['CONV_64_3x3_NOL2', 'r1', 'r0'], 
    ['CONV_128_5x5_KR00001', 'r2', 'r3'], 
    ['DROPOUT_0.3_SEED0', 'r4', 'r1'], 
    ['AVGPOOL_3x3', 'r1', 'r1'], 
    ['CONV_64_5x5_KR00001', 'r4', 'r2'], 
    ['AVGPOOL_5x5', 'r4', 'r2'], 
    ['CONV_128_5x5_KR01', 'r3', 'r0'], 
    ['CONV_32_3x3_KR00001_TANH', 'r1', 'r1'], 
    ['BATCH_NORM_1', 'r0', 'r2'],
    ]

    gp_tree = GP_tree()
    gp_tree.arity = 2

    print("")

    print("Original unmodified")

    print(gp_tree.simplify(indiv_test))

    repair_gene = "CONV_128_3x3_NOL2"

    new_indiv = repairInitialLayer(indiv_test, repair_gene)

    print("")

    print("Repair First Layer")

    print(gp_tree.simplify(new_indiv))

    model, ind_simplified, ind, repair_stats = generateModel(gp_tree.simplify(new_indiv), new_indiv, 1, a, b, gp_tree)

    print("")

    print("Repair First Layer")

    print(ind_simplified)




main()