diff --git a/README.md b/README.md index e901cb3..87f8fe5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -# Google Summer of Code 2021 -This GitHub repo has been developed from scratch by Özgür Kara under GSoC 2021 organization. It consists of the main program and it's [detailed documentation](https://github.com/NISYSLAB/VisualRLComposer/blob/main/documentation.pdf). If there is any issue, you can contact me via ozgurrkara99@gmail.com +# Google Summer of Code 2021 & 2022 +This GitHub repo has been developed from scratch by Özgür Kara under GSoC 2021 and has been extended by Mehul Sinha under GSoC 2022. It consists of the main program and it's [detailed documentation](https://github.com/NISYSLAB/VisualRLComposer/blob/main/documentation.pdf). If there is any issue, you can contact us via ozgurrkara99@gmail.com or mehulsinha73@gmail.com # VisualRLComposer The project aims to develop a GUI for facilitating the experimentation of Reinforcement Learning for the users. Particularly, researchers are able to test and implement their ideas and algorithms of reinforcement learning with the GUI easily even though they are not proficient in coding. @@ -8,9 +8,14 @@ The project aims to develop a GUI for facilitating the experimentation of Reinfo * PyQt5 based open source Graphical User Interface for visually testing the RL agents * Dragging and dropping the components allow users to create flows easily * Flows can be saved (in json format), loaded and new graphs can be opened using toolbar options +* Flows can also be created in DHGWorkflow, saved in GraphML format and loaded in VisualRLComposer as a new graph +* Users can create multiple Workflow graphs by opening a new scene tab * During testing, the relevant values such as rewards, states and actions are updated in a real-time manner -* There are six built-in RL environments and reward functions from OpenAI Gym and six RL agents that are imported from stable-baselines3 library +* There are seven built-in RL environments and reward functions from OpenAI Gym and six RL agents that are imported from stable-baselines3 library * Program allows users to both perform training and testing. Also, users can save their trained models as well as load their pretrained models according to their preferences +* Users can design custom policies for their RL models using the custom policy network designer +* The program allows users to train or test models with multi-environment experimentation in parallel +* The program allows user to keep track of model training via Tensorboard * Users are able to integrate their custom environments and reward functions to the program by following the procedure in the documentation * Detailed documentation and demo videos are provided in the GitHub page diff --git a/logo.png b/assets/logo.png similarity index 100% rename from logo.png rename to assets/logo.png diff --git a/documentation.pdf b/documentation.pdf index 1cb4433..e3ae195 100644 Binary files a/documentation.pdf and b/documentation.pdf differ diff --git a/main.py b/main.py index d02ceea..ede991c 100644 --- a/main.py +++ b/main.py @@ -1,12 +1,14 @@ -from PyQt5.QtWidgets import * +from PyQt5.QtWidgets import QApplication +from PyQt5.QtGui import QFont import sys +import os from rlcomposer.main_window import RLMainWindow if __name__ == '__main__': - import os - os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" app = QApplication(sys.argv) + # app.setStyle('QtCurve') + app.setFont(QFont("Helvetica", 9)) wnd = RLMainWindow() sys.exit(app.exec_()) - print("Exited") \ No newline at end of file + print("Exited") diff --git a/requirements.txt b/requirements.txt index aac25c4..4957815 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,12 @@ altgraph==0.17 atari-py==0.2.9 +swig +box2d-py==2.3.8 cloudpickle==1.6.0 cycler==0.10.0 future==0.18.2 gym==0.18.3 +gym-sokoban==0.0.6 joblib==1.0.1 kiwisolver==1.3.1 matplotlib==3.4.2 @@ -30,5 +33,6 @@ scipy==1.6.3 six==1.16.0 stable-baselines==2.10.0 stable-baselines3==1.0 +tensorboard==2.9.1 torch==1.9.0 typing-extensions==3.10.0.0 diff --git a/rlcomposer/custom_network_widget.py b/rlcomposer/custom_network_widget.py index 4f9c549..7bda3cc 100644 --- a/rlcomposer/custom_network_widget.py +++ b/rlcomposer/custom_network_widget.py @@ -1,18 +1,57 @@ from PyQt5 import QtWidgets, QtGui, QtCore, QtSvg -# from stadium.settings import GlobalConfig as config import os -# from stadium.utils import Dense, Conv2D, Model, Flatten +from rlcomposer.draw_nn import Dense, Conv2D, Model, Flatten, MaxPooling2D, AveragePooling2D # from stadium.core.defaults import CustomCnnPolicy, CustomMlpPolicy - - -# def get_icon(name): -# icon_path = os.path.join(config.ICONS, name + '.svg') -# return icon_path +import torch as th +import torch.nn as nn +from stable_baselines3.common.torch_layers import BaseFeaturesExtractor + + +class CustomCNN(BaseFeaturesExtractor): + """ + :param observation_space: (gym.Space) + :param features_dim: (int) Number of features extracted. + This corresponds to the number of unit for the last layer. + """ + + def __init__(self, observation_space, data, features_dim: int = 256): + super(CustomCNN, self).__init__(observation_space, features_dim) + # We assume CxHxW images (channels first) + # Re-ordering will be done by pre-preprocessing or wrapper + self.feature_dimension = features_dim + self.observation_space = observation_space + + n_input_channels = observation_space.shape[0] + self.cnn = nn.Sequential( + nn.Conv2d(n_input_channels, 32, kernel_size=8, stride=4, padding=0), + nn.ReLU(), + nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=0), + nn.ReLU(), + nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=0), + nn.ReLU(), + nn.Flatten(), + ) + self.cnn = data + with th.no_grad(): + n_flatten = self.cnn( + th.as_tensor(self.observation_space.sample()[None]).float() + ).shape[1] + + self.linear = nn.Sequential(nn.Linear(n_flatten, self.feature_dimension), nn.ReLU()) + + def forward(self, observations: th.Tensor) -> th.Tensor: + return self.linear(self.cnn(observations)) + + +class CallerSignal(QtCore.QObject): + signal = QtCore.pyqtSignal(str) class NetConfigWidget(QtWidgets.QWidget): def __init__(self, parent, name, config=None): super(NetConfigWidget, self).__init__(parent=parent) + self.signal = CallerSignal() + self.signal.signal.connect(self.caller) self.par = parent self.layers = [] self.config = config @@ -31,36 +70,46 @@ def __init__(self, parent, name, config=None): self.main_lay.addWidget(self.display) self.main_lay.addWidget(self.scroll) - def build(self, config, nn_type='CNN'): + def caller(self, nn_type): + self.build(nn_type) - self.combo = NewLayer(self) + def build(self, nn_type='Mlp'): + + self.combo = QtWidgets.QGridLayout(self) self.layers = [] self.container = QtWidgets.QWidget(self) self.lay = QtWidgets.QVBoxLayout(self.container) self.lay.addStretch() self.scroll.setWidget(self.container) - self.config = config if 'Cnn' in nn_type: self.flat = False - self.combo.model().item(2).setEnabled(True) - for i, n_filters in enumerate(config.filters): - layer = Conv( - parent=self, - filters=n_filters, - kernel=config.kernel_size[i], - stride=config.stride[i], - n=i) - self.add_layer(layer, update=False) + self.add_layer(Conv(parent=self, filters=16, kernel=4, stride=2, n=0), update=False) + self.add_layer(Pool(parent=self, kernel=2, stride=2, n=0), update=False) + self.add_layer(Conv(parent=self, filters=32, kernel=2, stride=2, n=1), update=False) else: self.flat = True - self.combo.model().item(2).setEnabled(False) - for i, nodes in enumerate(config.layers): + for i, nodes in enumerate([64]): layer = FC(self, nodes, n=i) self.add_layer(layer, update=False) - self.lay.addWidget(self.combo) + self.conv_layer_button = NewConvLayer(self) + self.pool_layer_button = NewPoolLayer(self) + self.activation_button = NewActivationLayer(self) + self.fc_layer_button = NewFCLayer(self) + self.enable_conf_button = EnabledToggle(self) + + self.combo.addWidget(self.conv_layer_button, 0, 0) + self.combo.addWidget(self.pool_layer_button, 0, 1) + self.combo.addWidget(self.activation_button, 0, 2) + self.combo.addWidget(self.fc_layer_button, 0, 3) + self.combo.addWidget(self.enable_conf_button, 0, 4) + if self.flat: + self.conv_layer_button.setEnabled(False) + self.pool_layer_button.setEnabled(False) + + self.lay.addLayout(self.combo) self.lay.addStretch() self.update_image() self.initialized = True @@ -68,12 +117,12 @@ def build(self, config, nn_type='CNN'): def add_layer(self, obj, update=True): index = len(self.layers) - if type(obj) is Conv and update: + if type(obj) in [Conv, Pool] and update: index = [type(x) is FC for x in self.layers].index(True) obj.update(index) self.lay.insertWidget(index, obj) self.layers.insert(index, obj) - # print([layer for layer in self.layers], index) + print([layer for layer in self.layers], index) if update: self.update_image() @@ -84,29 +133,29 @@ def delete_layer(self, layer): self.update_image() print([layer for layer in self.layers]) - # def update_image(self): - # action_space = self.par.manager.env.get_attr('action_space')[0] - # if hasattr(action_space, 'n'): - # n_outputs = action_space.n - # else: - # n_outputs = action_space.shape[0] - # input_shape = self.par.manager.env.get_attr('observation_space')[0].shape - # print(input_shape) - # if len(input_shape) <= 1: - # input_shape = (input_shape[0], 1, 1) - # imgpath = os.path.join(config.UTILS, 'net.svg') - # print(imgpath) - # model = Model(input_shape=input_shape) - # if self.flat: - # model.add(Flatten()) - # for i, layer in enumerate(self.layers): - # model.add(layer.to_draw()) - # if type(layer) is Conv and type(self.layers[i + 1]) is FC: - # model.add(Flatten()) - # model.add(Dense(n_outputs)) - # - # model.save_fig(imgpath) - # self.display.load(imgpath) + def update_image(self): + input_shape = self.par.getSpaceNames(self.par.instance.env_wrapper_list[0].env_name)[2] + output_shape = self.par.getSpaceNames(self.par.instance.env_wrapper_list[0].env_name)[3] + + if type(input_shape) is int: + input_shape = (input_shape, 1, 1) + + img_path = os.path.join('rlcomposer/rl/assets/net.svg') + self.model = Model(input_shape=input_shape) + if self.flat: + self.model.add(Flatten()) + for i, layer in enumerate(self.layers): + self.model.add(layer.to_draw()) + if type(layer) is Conv or type(layer) is Pool: + try: + if type(self.layers[i + 1]) is FC: + self.model.add(Flatten()) + except IndexError: + self.model.add(Flatten()) + + self.model.add(Dense(output_shape)) + self.model.save_fig(img_path) + self.display.load(img_path) # def blank(self): # imgpath = os.path.join(config.UTILS, 'blank.svg') @@ -114,27 +163,82 @@ def delete_layer(self, layer): def create_conf(self): conf = {} + cnn_list = [] if not self.flat: - filters, kernels, strides = [], [], [] - names = ['filters', 'kernel_size', 'stride'] + cnn_data = [] + prev_filter = None for layer in self.layers: if type(layer) is Conv: - filters += [layer.filters.val] - kernels += [layer.kernel.val] - strides += [layer.stride.val] - conf = {**dict(zip(names, [filters, kernels, strides]))} + prev_filter = layer.filters.val + cnn_data.append(dict({"filters": layer.filters.val, + "kernals": layer.kernel.val, + "strides": layer.stride.val, + "type": "Conv"})) + if type(layer) is Pool: + cnn_data.append(dict({"filters": prev_filter, + "kernals": layer.kernel.val, + "strides": layer.stride.val, + "type": layer.pool_type.val})) + + for i, layer in enumerate(cnn_data): + if i == 0 and layer["type"] == "Conv": + cnn_list.append(nn.Conv2d(in_channels=3, + out_channels=cnn_data[i]["filters"], + kernel_size=cnn_data[i]["kernals"], + stride=cnn_data[i]["strides"])) + if self.activation_button.function is None: + cnn_list.append(nn.ReLU()) + else: + cnn_list.append(self.activation_button.function()) + elif layer["type"] == "Conv": + cnn_list.append(nn.Conv2d(in_channels=cnn_data[i - 1]["filters"], + out_channels=cnn_data[i]["filters"], + kernel_size=cnn_data[i]["kernals"], + stride=cnn_data[i]["strides"])) + if self.activation_button.function is None: + cnn_list.append(nn.ReLU()) + else: + cnn_list.append(self.activation_button.function()) + + elif layer["type"] is MaxPooling2D: + cnn_list.append(nn.MaxPool2d(kernel_size=cnn_data[i]["kernals"], + stride=cnn_data[i]["strides"])) + elif layer["type"] is AveragePooling2D: + cnn_list.append(nn.AvgPool2d(kernel_size=cnn_data[i]["kernals"], + stride=cnn_data[i]["strides"])) + cnn_list.append(nn.Flatten()) fc_layers = [] for layer in self.layers: if type(layer) is FC: fc_layers.append(layer.nodes.val) - conf = {**conf, **{'layers': fc_layers}} - return conf - # if self.flat: - # return CustomMlpPolicy(**conf) - # else: - # return CustomCnnPolicy(**conf) + if self.flat: + if self.activation_button.function is None: + conf = dict( + net_arch=fc_layers + ) + else: + conf = dict( + net_arch=fc_layers, + activation_fn=self.activation_button.function + ) + else: + if self.activation_button.function is None: + conf = dict( + features_extractor_class=CustomCNN, + features_extractor_kwargs=dict(features_dim=fc_layers[0], data=nn.Sequential(*cnn_list)), + net_arch=fc_layers + ) + else: + conf = dict( + features_extractor_class=CustomCNN, + features_extractor_kwargs=dict(features_dim=fc_layers[0], data=nn.Sequential(*cnn_list)), + net_arch=fc_layers, + activation_fn=self.activation_button.function + ) + + return dict({"enabled": self.enable_conf_button.isChecked(), "conf": conf}) class ClickButton(QtWidgets.QPushButton): @@ -152,7 +256,7 @@ def __init__(self, parent, name, triggers, status=None, text=False): # icon.addPixmap(QtGui.QPixmap(path), size=QtCore.QSize(30, 30)) # self.setIcon(icon) if text: - # self.setStyleSheet("text-align:left;padding:4px;") + self.setStyleSheet("text-align:left;padding:4px;") self.setText(' ' + name) @@ -181,6 +285,32 @@ def val(self): return self.spin.value() +class ComboBox(QtWidgets.QWidget): + def __init__(self, par, name, val): + super(ComboBox, self).__init__() + self.par = par + self.label = QtWidgets.QLabel() + self.label.setText(name) + self.mapping = dict({'MaxPooling': MaxPooling2D, 'AveragePooling': AveragePooling2D}) + self.combo = QtWidgets.QComboBox() + self.combo.addItems(self.mapping.keys()) + self.combo.setCurrentText(val) + font = self.label.font() + font.setPointSize(font.pointSize() - 2) + self.combo.setFont(font) + self.label.setFont(font) + self.lay = QtWidgets.QHBoxLayout(self) + self.lay.setContentsMargins(-1, 0, -1, 0) + self.lay.setSpacing(0) + self.lay.addWidget(self.label) + self.lay.addWidget(self.combo) + self.combo.currentIndexChanged.connect(self.par.update_image) + + @property + def val(self): + return self.mapping[self.combo.currentText()] + + class Layer(QtWidgets.QWidget): def __init__(self, parent, n=0): super(Layer, self).__init__(parent) @@ -189,6 +319,7 @@ def __init__(self, parent, n=0): self.n = n self.label = QtWidgets.QLabel(self.type + ':' + str(self.n)) self.del_button = ClickButton(self, 'Delete', [self.delete], status='Delete Layer') + self.del_button.setIcon(QtGui.QIcon('rlcomposer/rl/assets/delete.svg')) self.del_button.setFixedSize(30, 30) self.lay = QtWidgets.QHBoxLayout(self) self.lay.setContentsMargins(-1, 0, -1, 0) @@ -210,55 +341,124 @@ def update(self, n): class Conv(Layer): - def __init__(self, parent, filters=64, kernel=3, stride=1, padding='valid', n=0): + def __init__(self, parent, filters=64, kernel=3, stride=1, padding='same', n=0): super(Conv, self).__init__(parent, n) self.filters = DialSpin(self.par, 'Filters:', 512, val=filters) self.kernel = DialSpin(self.par, 'Kernel:', 15, val=kernel) - self.stride = DialSpin(self.par, 'Stride:', 3, val=stride) + self.stride = DialSpin(self.par, 'Stride:', 4, val=stride) self.padding = padding self.lay.addWidget(self.filters) + self.lay.addWidget(self.kernel) + self.lay.addWidget(self.stride) + self.lay.addWidget(self.del_button) + + def to_draw(self): + k, s = self.kernel.val, self.stride.val + lay = Conv2D(filters=self.filters.val, kernel_size=(k, k), strides=(s, s), padding=self.padding) + return lay + +class Pool(Layer): + def __init__(self, parent, kernel=1, stride=1, n=0): + super(Pool, self).__init__(parent, n) + self.pool_type = ComboBox(self.par, 'Pooling', val='MaxPooling') + self.kernel = DialSpin(self.par, 'Kernel:', 15, val=kernel) + self.stride = DialSpin(self.par, 'Stride:', 4, val=stride) + + self.lay.addWidget(self.pool_type) self.lay.addWidget(self.kernel) self.lay.addWidget(self.stride) self.lay.addWidget(self.del_button) - # def to_draw(self): - # k, s = self.kernel.val, self.stride.val - # lay = Conv2D(filters=self.filters.val, kernel_size=(k, k), strides=(s, s), padding=self.padding) - # return lay + def to_draw(self): + k, s, obj = self.kernel.val, self.stride.val, self.pool_type.val + lay = obj(pool_size=(k, k), strides=(s, s)) + return lay class FC(Layer): - def __init__(self, parent, nodes=128, n=0): + def __init__(self, parent, nodes=64, n=0): super(FC, self).__init__(parent, n) self.nodes = DialSpin(self.par, 'Nodes:', 2048, val=nodes) self.lay.addWidget(self.nodes) self.lay.addWidget(self.del_button) - # def to_draw(self): - # lay = Dense(units=self.nodes.val) - # return lay + def to_draw(self): + lay = Dense(units=self.nodes.val) + return lay -class NewLayer(QtWidgets.QComboBox): +class NewFCLayer(QtWidgets.QPushButton): def __init__(self, parent): - super(NewLayer, self).__init__() + super(NewFCLayer, self).__init__() self.par = parent - self.options = ['New Layer', 'Fully Connected', 'Convolution'] - self.mapping = dict(zip(self.options, [None, FC, Conv])) + self.clicked.connect(self.new_FC_layer) + self.setText("Fully Connected Layer") + + def new_FC_layer(self): + self.par.add_layer(FC(self.par)) + + +class NewConvLayer(QtWidgets.QPushButton): + def __init__(self, parent): + super(NewConvLayer, self).__init__() + self.par = parent + self.clicked.connect(self.new_Conv_layer) + self.setText("Convolutional Layer") + + def new_Conv_layer(self): + self.par.add_layer(Conv(self.par)) + + +class NewPoolLayer(QtWidgets.QPushButton): + def __init__(self, parent): + super(NewPoolLayer, self).__init__() + self.par = parent + self.clicked.connect(self.new_Pool_layer) + self.setText("Pooling Layer") + + def new_Pool_layer(self): + self.par.add_layer(Pool(self.par)) + + +class NewActivationLayer(QtWidgets.QComboBox): + def __init__(self, parent): + super(NewActivationLayer, self).__init__() + self.par = parent + self.function = None + self.mapping = dict({'Activation Function': None, + 'ReLU': nn.ReLU, + 'Tanh': nn.Tanh, + 'Sigmoid': nn.Sigmoid, + 'ELU': nn.ELU, + 'GLU': nn.GLU, + 'Softmin': nn.Softmin, + 'Softmax': nn.Softmax}) self.activated.connect(self.new_layer) self.add_options() def new_layer(self): i = self.currentIndex() - 1 object = self.mapping[self.currentText()] - self.par.add_layer(object(self.par)) - self.setCurrentIndex(0) - - # def add_options(self): - # for option in self.options: - # icon = QtGui.QIcon(QtGui.QPixmap(get_icon('Default'))) - # self.addItem(icon, option) - # self.model().item(0).setEnabled(False) \ No newline at end of file + self.function = object + + def add_options(self): + for option in self.mapping.keys(): + self.addItem(option) + self.model().item(0).setEnabled(False) + + +class EnabledToggle(QtWidgets.QPushButton): + def __init__(self, parent): + super(EnabledToggle, self).__init__() + self.setCheckable(True) + self.clicked.connect(self.changeState) + self.changeState() + + def changeState(self): + if self.isChecked(): + self.setText("Enabled") + else: + self.setText("Disabled") diff --git a/rlcomposer/draw_nn.py b/rlcomposer/draw_nn.py new file mode 100644 index 0000000..e6e53b6 --- /dev/null +++ b/rlcomposer/draw_nn.py @@ -0,0 +1,396 @@ +import math +from abc import ABCMeta, abstractmethod + +import math +class Config: + theta = - math.pi / 6 + ratio = 0.7 + bounding_box_margin = 10 + inter_layer_margin = 50 + text_margin = 10 + channel_scale = 3 / 5 + text_size = 8 + one_dim_width = 4 + line_color_feature_map = (0, 0, 0) + line_color_layer = (0, 0, 255) + text_color_feature_map = (0, 0, 0) + text_color_layer = (0, 0, 0) + +config = Config() + +class Line: + def __init__(self, x1, y1, x2, y2, color=(0, 0, 0), width=1, dasharray=None): + self.x1, self.y1 = x1, y1 + self.x2, self.y2 = x2, y2 + self.color = color + self.width = width + self.dasharray = dasharray + + def get_svg_string(self): + stroke_dasharray = self.dasharray if self.dasharray else "none" + return '\n'.format( + self.x1, self.y1, self.x2, self.y2, self.width, stroke_dasharray, self.color) + + +class Text: + def __init__(self, x, y, body, color=(0, 0, 0), size=10): + self.x = x + self.y = y + self.body = body + self.color = color + self.size = size + + def get_svg_string(self): + return '{}\n'.format(self.x, self.y, self.size, self.color, self.body) + + +class Model: + def __init__(self, input_shape): + self.layers = [] + + if len(input_shape) != 3: + raise ValueError("input_shape should be rank 3 but received {}".format(input_shape)) + + self.feature_maps = [] + self.x = None + self.y = None + self.width = None + self.height = None + + self.feature_maps.append(FeatureMap3D(*input_shape)) + + def add_feature_map(self, layer): + if isinstance(self.feature_maps[-1], FeatureMap3D): + h, w = self.feature_maps[-1].h, self.feature_maps[-1].w + filters = layer.filters if layer.filters else self.feature_maps[-1].c + + if isinstance(layer, GlobalAveragePooling2D): + self.feature_maps.append(FeatureMap1D(filters)) + elif isinstance(layer, Flatten): + self.feature_maps.append(FeatureMap1D(h * w * filters)) + elif isinstance(layer, Deconv2D): + if layer.padding == "same": + new_h = h * layer.strides[0] + new_w = w * layer.strides[1] + else: + new_h = h * layer.strides[0] + max(layer.kernel_size[0] - layer.strides[0], 0) + new_w = w * layer.strides[1] + max(layer.kernel_size[1] - layer.strides[1], 0) + self.feature_maps.append(FeatureMap3D(new_h, new_w, filters)) + else: + if layer.padding == "same": + new_h = math.ceil(h / layer.strides[0]) + new_w = math.ceil(w / layer.strides[1]) + else: + new_h = math.ceil((h - layer.kernel_size[0] + 1) / layer.strides[0]) + new_w = math.ceil((w - layer.kernel_size[1] + 1) / layer.strides[1]) + self.feature_maps.append(FeatureMap3D(new_h, new_w, filters)) + else: + self.feature_maps.append(FeatureMap1D(layer.filters)) + + def add(self, layer): + self.add_feature_map(layer) + layer.prev_feature_map = self.feature_maps[-2] + layer.next_feature_map = self.feature_maps[-1] + self.layers.append(layer) + + def build(self): + left = 0 + + for feature_map in self.feature_maps: + right = feature_map.set_objects(left) + left = right + config.inter_layer_margin + + for i, layer in enumerate(self.layers): + layer.set_objects() + + # get bounding box + self.x = - config.bounding_box_margin - 30 + self.y = min([f.get_top() for f in self.feature_maps]) - config.text_margin - config.text_size \ + - config.bounding_box_margin + self.width = self.feature_maps[-1].right + config.bounding_box_margin * 2 + 30 * 2 + # TODO: automatically calculate the ad-hoc offset "30" from description length + self.height = - self.y * 2 + config.text_size + + def save_fig(self, filename): + self.build() + string = '\n'.format(self.x, self.y, self.width, self.height) + + for feature_map in self.feature_maps: + string += feature_map.get_object_string() + + for layer in self.layers: + string += layer.get_object_string() + + string += '' + f = open(filename, 'w') + f.write(string) + f.close() + + +class FeatureMap: + __metaclass__ = ABCMeta + + def __init__(self): + self.left = None + self.right = None + self.objects = None + + @abstractmethod + def set_objects(self, left): + pass + + def get_object_string(self): + return get_object_string(self.objects) + + @abstractmethod + def get_top(self): + pass + + @abstractmethod + def get_bottom(self): + pass + + +class FeatureMap3D(FeatureMap): + def __init__(self, h, w, c): + self.h = h + self.w = w + self.c = c + super(FeatureMap3D, self).__init__() + + def set_objects(self, left): + self.left = left + c_ = math.pow(self.c, config.channel_scale) + self.right, self.objects = get_rectangular(self.h, self.w, c_, left, config.line_color_feature_map) + x = (left + self.right) / 2 + y = self.get_top() - config.text_margin + self.objects.append(Text(x, y, "{}x{}x{}".format(self.h, self.w, self.c), color=config.text_color_feature_map, + size=config.text_size)) + + return self.right + + def get_left_for_conv(self): + return self.left + self.w * config.ratio * math.cos(config.theta) / 2 + + def get_top(self): + return - self.h / 2 + self.w * config.ratio * math.sin(config.theta) / 2 + + def get_bottom(self): + return self.h / 2 - self.w * config.ratio * math.sin(config.theta) / 2 + + def get_right_for_conv(self): + x = self.left + self.w * config.ratio * math.cos(config.theta) / 4 + y = - self.h / 4 + self.w * config.ratio * math.sin(config.theta) / 4 + + return x, y + + +class FeatureMap1D(FeatureMap): + def __init__(self, c): + self.c = c + super(FeatureMap1D, self).__init__() + + def set_objects(self, left): + self.left = left + c_ = math.pow(self.c, config.channel_scale) + self.right = left + config.one_dim_width + # TODO: reflect text length to right + x1 = left + y1 = - c_ / 2 + x2 = left + config.one_dim_width + y2 = c_ / 2 + line_color = config.line_color_feature_map + self.objects = [] + self.objects.append(Line(x1, y1, x1, y2, line_color)) + self.objects.append(Line(x1, y2, x2, y2, line_color)) + self.objects.append(Line(x2, y2, x2, y1, line_color)) + self.objects.append(Line(x2, y1, x1, y1, line_color)) + self.objects.append(Text(left + config.one_dim_width / 2, - c_ / 2 - config.text_margin, "{}".format( + self.c), color=config.text_color_feature_map, size=config.text_size)) + + return self.right + + def get_top(self): + return - math.pow(self.c, config.channel_scale) / 2 + + def get_bottom(self): + return math.pow(self.c, config.channel_scale) / 2 + + +class Layer: + __metaclass__ = ABCMeta + + def __init__(self, filters=None, kernel_size=None, strides=(1, 1), padding="valid"): + self.filters = filters + self.kernel_size = kernel_size + self.strides = strides + self.padding = padding + self.objects = [] + self.prev_feature_map = None + self.next_feature_map = None + self.description = None + + @abstractmethod + def get_description(self): + return None + + def set_objects(self): + c = math.pow(self.prev_feature_map.c, config.channel_scale) + left = self.prev_feature_map.get_left_for_conv() + start1 = (left + c, + -self.kernel_size[0] + self.kernel_size[1] * config.ratio * math.sin(config.theta) / 2 + + self.kernel_size[0] / 2) + start2 = (left + c + self.kernel_size[1] * config.ratio * math.cos(config.theta), + -self.kernel_size[1] * config.ratio * math.sin(config.theta) / 2 + self.kernel_size[0] / 2) + end = self.next_feature_map.get_right_for_conv() + line_color = config.line_color_layer + left, self.objects = get_rectangular(self.kernel_size[0], self.kernel_size[1], c, left, color=line_color) + self.objects.append(Line(start1[0], start1[1], end[0], end[1], color=line_color)) + self.objects.append(Line(start2[0], start2[1], end[0], end[1], color=line_color)) + + x = (self.prev_feature_map.right + self.next_feature_map.left) / 2 + y = max(self.prev_feature_map.get_bottom(), self.next_feature_map.get_bottom()) + config.text_margin \ + + config.text_size + + for i, description in enumerate(self.get_description()): + self.objects.append(Text(x, y + i * config.text_size, "{}".format(description), + color=config.text_color_layer, size=config.text_size)) + + def get_object_string(self): + return get_object_string(self.objects) + + +class Conv2D(Layer): + def get_description(self): + return ["conv{}x{}, {}".format(self.kernel_size[0], self.kernel_size[1], self.filters), + "stride {}".format(self.strides)] + + +class Deconv2D(Layer): + def get_description(self): + return ["deconv{}x{}, {}".format(self.kernel_size[0], self.kernel_size[1], self.filters), + "stride {}".format(self.strides)] + + +class PoolingLayer(Layer): + def __init__(self, pool_size=(2, 2), strides=None, padding="same"): + if not strides: + strides = pool_size + super(PoolingLayer, self).__init__(kernel_size=pool_size, strides=strides, padding=padding) + + +class AveragePooling2D(PoolingLayer): + def get_description(self): + return ["avgpool{}x{}".format(self.kernel_size[0], self.kernel_size[1]), + "stride {}".format(self.strides)] + + +class MaxPooling2D(PoolingLayer): + def get_description(self): + return ["maxpool{}x{}".format(self.kernel_size[0], self.kernel_size[1]), + "stride {}".format(self.strides)] + + +class GlobalAveragePooling2D(Layer): + def __init__(self): + super(GlobalAveragePooling2D, self).__init__() + + def get_description(self): + return ["global avepool"] + + def set_objects(self): + x = (self.prev_feature_map.right + self.next_feature_map.left) / 2 + y = max(self.prev_feature_map.get_bottom(), self.next_feature_map.get_bottom()) + config.text_margin \ + + config.text_size + + for i, description in enumerate(self.get_description()): + self.objects.append(Text(x, y + i * config.text_size, "{}".format(description), + color=config.text_color_layer, size=config.text_size)) + + +class Flatten(Layer): + def __init__(self): + super(Flatten, self).__init__() + + def get_description(self): + return ["flatten"] + + def set_objects(self): + x = (self.prev_feature_map.right + self.next_feature_map.left) / 2 + y = max(self.prev_feature_map.get_bottom(), self.next_feature_map.get_bottom()) + config.text_margin \ + + config.text_size + + for i, description in enumerate(self.get_description()): + self.objects.append(Text(x, y + i * config.text_size, "{}".format(description), + color=config.text_color_layer, size=config.text_size)) + + +class Dense(Layer): + def __init__(self, units): + super(Dense, self).__init__(filters=units) + + def get_description(self): + return ["dense"] + + def set_objects(self): + x1 = self.prev_feature_map.right + y11 = - math.pow(self.prev_feature_map.c, config.channel_scale) / 2 + y12 = math.pow(self.prev_feature_map.c, config.channel_scale) / 2 + x2 = self.next_feature_map.left + y2 = - math.pow(self.next_feature_map.c, config.channel_scale) / 4 + line_color = config.line_color_layer + self.objects.append(Line(x1, y11, x2, y2, color=line_color, dasharray=2)) + self.objects.append(Line(x1, y12, x2, y2, color=line_color, dasharray=2)) + + x = (self.prev_feature_map.right + self.next_feature_map.left) / 2 + y = max(self.prev_feature_map.get_bottom(), self.next_feature_map.get_bottom()) + config.text_margin \ + + config.text_size + + for i, description in enumerate(self.get_description()): + self.objects.append(Text(x, y + i * config.text_size, "{}".format(description), + color=config.text_color_layer, size=config.text_size)) + + +def get_rectangular(h, w, c, dx=0, color=(0, 0, 0)): + p = [[0, -h], + [w * config.ratio * math.cos(config.theta), -w * config.ratio * math.sin(config.theta)], + [c, 0]] + + dy = w * config.ratio * math.sin(config.theta) / 2 + h / 2 + right = dx + w * config.ratio * math.cos(config.theta) + c + lines = [] + + for i, [x1, y1] in enumerate(p): + for x2, y2 in [[0, 0], p[(i + 1) % 3]]: + for x3, y3 in [[0, 0], p[(i + 2) % 3]]: + lines.append(Line(x2 + x3 + dx, y2 + y3 + dy, x1 + x2 + x3 + dx, y1 + y2 + y3 + dy, + color=color)) + + for i in [1, 6, 8]: + lines[i].dasharray = 1 + + return right, lines + + +def get_object_string(objects): + return "".join([obj.get_svg_string() for obj in objects]) + + +def main(): + model = Model(input_shape=(128, 128, 3)) + model.add(Conv2D(32, (11, 11), (2, 2), padding="same")) + model.add(MaxPooling2D((2, 2))) + model.add(Conv2D(64, (7, 7), padding="same")) + model.add(AveragePooling2D((2, 2))) + model.add(Conv2D(128, (3, 3), padding="same")) + model.add(MaxPooling2D((2, 2))) + model.add(Conv2D(256, (3, 3), padding="same")) + model.add(Conv2D(512, (3, 3), padding="same")) + model.save_fig("test.svg") + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/rlcomposer/interface.py b/rlcomposer/interface.py index f9635c0..8e9bf8b 100644 --- a/rlcomposer/interface.py +++ b/rlcomposer/interface.py @@ -2,22 +2,23 @@ from PyQt5.QtWidgets import * from PyQt5.QtGui import * -from PyQt5 import QtCore - from .window_widget import RLComposerWindow from .tensorboard_widget import Tensorboard -from .plot_widget import WidgetPlot +from .plot_widget import TestingWidgetPlot, TrainingWidgetPlot from .custom_network_widget import NetConfigWidget from .treeview_widget import FunctionTree -import numpy as np from .rl.instance import Instance +from .plot_button import PlotButton import os -from .test_plots import TestPlots +import numpy as np DEBUG = True -class InstanceWorker(QRunnable): +class InstanceWorkerSignals(QObject): + finished = pyqtSignal() + +class InstanceWorker(QRunnable): def __init__(self, fn, start_fn, stop_fn): super(InstanceWorker, self).__init__() self.continue_run = True # provide a bool run condition for the class @@ -26,19 +27,20 @@ def __init__(self, fn, start_fn, stop_fn): self.fn = fn self.start_fn = start_fn self.stop_fn = stop_fn - self.signals = WorkerSignals() - - + self.signals = InstanceWorkerSignals() def run(self): i = 0 - self.start_fn() + #try: + # self.start_fn() + #except Exception as e: + # print(e) + # self.stop_fn() while self.start_run: QThread.msleep(0) while self.continue_run: # give the loop a stoppable condition self.fn(i) - # QThread.msleep(0) i = i + 1 while self.pause_f: QThread.msleep(0) @@ -59,18 +61,40 @@ def stop(self): self.continue_run = False # set the run condition to false on stop print("Finish signal emitted") -class Worker(QRunnable): +class WorkerSignals(QObject): + progress = pyqtSignal(int) + url = pyqtSignal(str) + finished_value = True + + +class Worker(QRunnable): def __init__(self, fn): super(Worker, self).__init__() + self.signals = WorkerSignals() self.fn = fn @pyqtSlot() def run(self): - self.fn() + self.fn(self.signals) -class WorkerSignals(QObject): - finished = pyqtSignal() + def stop(self): + self.signals.finished_value = False + + +class Plot(QRunnable): + def __init__(self, fn): + super(Plot, self).__init__() + self.fn = fn + self.start_run = True + + @pyqtSlot() + def run(self): + while self.start_run: + self.fn.refresh() + + def stop(self): + self.start_run = False class Interface(QWidget): @@ -94,29 +118,38 @@ def initUI(self): self.tensorboard = Tensorboard() - self.raw_plot_widget = WidgetPlot(name="Reward") - self.state_plot_widget = WidgetPlot(name="State") - self.action_plot_widget = WidgetPlot(name="Action") + self.testing_reward_widget = TestingWidgetPlot(name="Reward") + self.testing_state_widget = TestingWidgetPlot(name="State") + self.testing_action_widget = TestingWidgetPlot(name="Action") + self.training_reward_widget = TrainingWidgetPlot(name="Reward") + self.training_action_widget = TrainingWidgetPlot(name="Action") + + self.tree = FunctionTree(self.window_widget) self.netconf = NetConfigWidget(self, '') self.plot_tab = QTabWidget(self) - self.test_plot_widgets = TestPlots(self.raw_plot_widget, self.action_plot_widget, self.state_plot_widget) + # self.test_plot_widgets = TestPlots(self.reward_plot_widget, self.action_plot_widget, self.state_plot_widget) self.plot_tab.addTab(self.tensorboard, 'Tensorboard') - self.plot_tab.addTab(self.test_plot_widgets, "Testing Plots") - # self.plot_tab.addTab(self.raw_plot_widget, "Rewards") - # self.plot_tab.addTab(self.state_plot_widget, "States") - # self.plot_tab.addTab(self.action_plot_widget, "Actions") - # self.plot_tab.addTab(self.netconf, "Custom Network") + self.plot_button_widgets = PlotButton(self.testing_reward_widget, self.testing_action_widget, self.testing_state_widget, + self.training_reward_widget, self.training_action_widget) + self.plot_button_widgets.set_training_buttons(False) + self.plot_button_widgets.set_testing_buttons(False) + self.plot_tab.addTab(self.plot_button_widgets, "Plots") + self.plot_tab.addTab(self.netconf, "Custom Network") self.plot_tab.currentChanged.connect(self.onTabChange) - self.tree = FunctionTree(self.window_widget.scene) + self.plot_tab_reload = QToolButton(self) + self.plot_tab_reload.setIcon(QIcon('rlcomposer/rl/assets/refresh.svg')) + self.plot_tab.setCornerWidget(self.plot_tab_reload) + self.plot_tab_reload.clicked.connect(self.tensorboard.reload) self.img_view = QLabel(self) self.data = np.random.rand(256, 256) qimage = QImage(self.data, self.data.shape[0], self.data.shape[1], QImage.Format_RGB32) self.pixmap = QPixmap(qimage) self.img_view.setPixmap(self.pixmap) + self.tree.status.setText("Status: Create an Instance") self.pauseButton = QPushButton("Pause/Continue", self) self.pauseButton.clicked.connect(self.pauseContinue) @@ -133,12 +166,10 @@ def initUI(self): self.trainButton.clicked.connect(self.trainThread) self.trainButton.setEnabled(False) - self.testButton = QPushButton("Test Instance", self) self.testButton.clicked.connect(self.testThread) self.testButton.setEnabled(False) - self.closeButton = QPushButton("Close Instance", self) self.closeButton.clicked.connect(self.closeInstanceButton) self.closeButton.setEnabled(False) @@ -166,7 +197,7 @@ def createLayout(self): layout.addWidget(self.img_view, 1, 1, 1, 6) layout.addWidget(self.createButton, 2, 1) layout.addWidget(self.trainButton, 2, 2) - layout.addWidget(self.saveModelButton, 2,3) + layout.addWidget(self.saveModelButton, 2, 3) layout.addWidget(self.testButton, 2, 4) layout.addWidget(self.pauseButton, 2, 5) layout.addWidget(self.closeButton, 2, 6) @@ -177,16 +208,32 @@ def threadComplete(self): self.threadpool.clear() def initInstance(self): + self.tree.status.setText("Status: Creating Instance") self.instance = Instance(self.window_widget.scene) + self.n_envs = len(self.instance.env_wrapper_list) img = self.instance.prep() self.img_view.setPixmap(self.convertToPixmap(img)) + self.tree.status.setText("Status: Instance Created") + self.createButton.setEnabled(False) + if not self.checkLoaded(): + self.trainButton.setEnabled(True) + self.testButton.setEnabled(True) + self.closeButton.setEnabled(True) + self.pauseButton.setEnabled(False) + + if type(self.getSpaceNames(self.instance.env_wrapper_list[0].env_name)[2]) is tuple: + self.netconf.signal.signal.emit('Cnn') + else: + self.netconf.signal.signal.emit('Mlp') def pauseContinue(self): if self.p: self.p = False + self.tree.status.setText("Status: Paused") self.test_worker.pause() else: self.p=True + self.tree.status.setText("Status: Testing in Progress") self.test_worker.cont() @@ -204,88 +251,172 @@ def checkLoaded(self): return False def createInstance(self): - self.raw_plot_widget.canvas.set_data() - self.state_plot_widget.canvas.set_data() - self.action_plot_widget.canvas.set_data() self.test_worker = InstanceWorker(self.testInstance, self.initInstance, self.closeInstance) self.test_worker.setAutoDelete(True) self.test_worker.signals.finished.connect(self.threadComplete) - self.createButton.setEnabled(False) - if not self.checkLoaded(): self.trainButton.setEnabled(True) - self.testButton.setEnabled(True) - self.closeButton.setEnabled(True) - self.pauseButton.setEnabled(True) self.threadpool.start(self.test_worker) + + self.testing_reward_thread = Plot(self.testing_reward_widget) + self.testing_state_thread = Plot(self.testing_state_widget) + self.testing_action_thread = Plot(self.testing_action_widget) + self.testing_reward_thread.setAutoDelete(True) + self.testing_state_thread.setAutoDelete(True) + self.testing_action_thread.setAutoDelete(True) + + self.training_reward_thread = Plot(self.training_reward_widget) + self.training_action_thread = Plot(self.training_action_widget) + self.training_reward_thread.setAutoDelete(True) + self.training_action_thread.setAutoDelete(True) def closeInstanceButton(self): self.saveModelButton.setEnabled(False) self.closeButton.setEnabled(False) self.testButton.setEnabled(False) + self.trainButton.setEnabled(False) self.pauseButton.setEnabled(False) self.createButton.setEnabled(True) - self.test_worker.cont() - self.test_worker.stop() - self.instance._tensorboard_kill() + self.plot_button_widgets.set_training_buttons(False) + self.plot_button_widgets.set_testing_buttons(False) - def closeInstance(self): - self.test_worker.stop() - self.instance.env.close() - del self.instance - del self.test_worker + self.tree.status.setText("Status: Create an Instance") self.img_view.setPixmap(self.pixmap) + self.tensorboard.load(QUrl()) + try: + self.training_reward_widget.clear_canvas() + self.training_action_widget.clear_canvas() + self.training_reward_thread.stop() + self.training_action_thread.stop() + except Exception as e: + print("Training Widget clearing error", e) - def trainInstance(self): - self.tensorboard.delayed_load() - self.instance.tensorboard(browser=False) + try: + self.testing_reward_widget.clear_canvas() + self.testing_action_widget.clear_canvas() + self.testing_state_widget.clear_canvas() + self.testing_reward_thread.stop() + self.testing_state_thread.stop() + self.testing_action_thread.stop() + except Exception as e: + print("Testing Widget clearing error", e) + + self.test_worker.pause() + self.test_worker.stop() + del self.test_worker + try: + self.worker.stop() + self.worker.signals.progress.emit(0) + del self.worker + except Exception as e: + print(e) - self.instance.train_model() + try: + self.instance._tensorboard_kill() + del self.instance + except Exception as e: + print(e) + + def closeInstance(self): + try: + self.instance.env.close() + self.initUI() + except Exception as e: + print(e) + print("Create a Scene first!") + self.createButton.setEnabled(True) + self.testButton.setEnabled(False) + self.closeButton.setEnabled(False) + self.pauseButton.setEnabled(False) + self.trainButton.setEnabled(False) + + def trainInstance(self, signal): + self.tree.status.setText("Status: Training in Progress") + plots = [self.training_reward_widget, self.training_action_widget] + self.instance.train_model(self.netconf.create_conf(), signal, plots) self.trainButton.setEnabled(False) self.saveModelButton.setEnabled(True) - pass + if not signal.finished_value: + signal.progress.emit(0) + self.saveModelButton.setEnabled(False) + self.tree.status.setText("Status: Training Finished") def testThread(self): + self.threadpool.start(self.testing_reward_thread) + self.threadpool.start(self.testing_state_thread) + self.threadpool.start(self.testing_action_thread) + self.testing_reward_widget.set_canvas(self.n_envs, ["Reward"]) + self.testing_state_widget.set_canvas(self.n_envs, self.getSpaceNames(self.instance.env_wrapper_list[0].env_name)[0]) + self.testing_action_widget.set_canvas(self.n_envs, self.getSpaceNames(self.instance.env_wrapper_list[0].env_name)[1]) + self.plot_button_widgets.set_testing_buttons(True) self.pauseButton.setEnabled(True) self.testButton.setEnabled(False) + self.tree.status.setText("Status: Testing in Progress") self.test_worker.start() def trainThread(self): - worker = Worker(self.trainInstance) - self.threadpool.start(worker) + #self.tensorboard.initial_load() + self.threadpool.start(self.training_reward_thread) + self.threadpool.start(self.training_action_thread) + self.training_reward_widget.set_canvas(self.n_envs, ["Reward"]) + self.training_action_widget.set_canvas(self.n_envs, self.getSpaceNames(self.instance.env_wrapper_list[0].env_name)[1]) + self.plot_button_widgets.set_training_buttons(True) + + self.worker = Worker(self.trainInstance) + self.worker.signals.progress.connect(self.tree.progress_bar_handler) + self.worker.signals.url.connect(self.tensorboard.setURL) + self.threadpool.start(self.worker) def testInstance(self, step): img, reward, done, action_probabilities, self.state, action = self.instance.step() self.img_view.setPixmap(self.convertToPixmap(img)) - print("step") - self.raw_plot_widget.canvas.update_plot(step, reward, ["Reward"]) - self.state_plot_widget.canvas.update_plot(step, self.state, self.getSpaceNames(self.instance.env_wrapper.env_name)[0]) - self.action_plot_widget.canvas.update_plot(step, action, self.getSpaceNames(self.instance.env_wrapper.env_name)[1]) + print(f"Step {step}") + self.testing_reward_widget.update_data(step, reward, ["Reward"]) + if self.getSpaceNames(self.instance.env_wrapper_list[0].env_name)[0] != "Invalid": + self.testing_state_widget.update_data(step, self.state, self.getSpaceNames(self.instance.env_wrapper_list[0].env_name)[0]) + self.testing_action_widget.update_data(step, action, self.getSpaceNames(self.instance.env_wrapper_list[0].env_name)[1]) def getSpaceNames(self, env_name): - state_label, action_label = [], [] + state_label, action_label, action_shape, observation_shape = [], [], None, None if env_name == "Pendulum": state_label = ["sin(theta)", "cos(theta)", "Velocity"] action_label = ["Torque"] + observation_shape, action_shape = 3, 1 elif env_name == "CartPoleEnv": state_label = ["Cart Position", "Cart Velocity", "Pole Angle", "Pole Angular Velocity"] action_label = ["0: Left, 1: Right"] + observation_shape, action_shape = 4, 2 elif env_name == "AcrobotEnv": state_label = ["cos(theta1)", "sin(theta1)", "cos(theta2)", "sin(theta2)", "Velocity of 1", "Velocity of 2"] action_label = ["Torque"] + observation_shape, action_shape = 6, 3 + elif env_name == "Continuous_MountainCarEnv": state_label = ["Position", "Velocity"] action_label = ["Action"] - else: - pass + observation_shape, action_shape = 2, 1 + + elif env_name == "MountainCarEnv": + state_label = ["Position", "Velocity"] + action_label = ["Action"] + observation_shape, action_shape = 2, 3 - return state_label, action_label + elif env_name == "LunarLander": + state_label = ["Coord-X", "Coord-Y", "Velocity-X", "Velocity-Y", "Angle", "Angular Velocity", "Left Leg Contact", "Right Leg Contact"] + action_label = ["0:Nothing, 1:Fire Left, 2:Fire Main, 3:Fire Right"] + observation_shape, action_shape = 8, 4 + elif env_name == "SokobanEnv": + state_label = ["Invalid"] + action_label = ["Action"] + observation_shape, action_shape = (160, 160, 3), 9 - def stepThread(self): - pass + else: + pass + + return state_label, action_label, observation_shape, action_shape def convertToPixmap(self, img): im = np.transpose(img, (0, 1, 2)).copy() @@ -294,8 +425,7 @@ def convertToPixmap(self, img): return pixmap def onTabChange(self, i): # changed! - return - + pass # def createTitle(self): # title = "Visual RL Composer - " diff --git a/rlcomposer/main_window.py b/rlcomposer/main_window.py index f3aed0e..c13be41 100644 --- a/rlcomposer/main_window.py +++ b/rlcomposer/main_window.py @@ -54,8 +54,8 @@ def initUI(self): self.setCentralWidget(self.widget) # set window properties - self.setGeometry(200, 200, 800, 600) - self.setWindowIcon(QIcon('logo.png')) + self.setWindowState(Qt.WindowMaximized) + self.setWindowIcon(QIcon('assets/logo.png')) self.show() def createTitle(self): @@ -84,7 +84,10 @@ def clickedFileOpen(self): if fname == "": return if os.path.isfile(fname): - self.window_widget.scene.loadFromFile(fname) + if fname.endswith('.graphml'): + self.window_widget.scene.loadFromGraphML(fname) + else: + self.window_widget.scene.loadFromFile(fname) self.fname = fname self.createTitle() diff --git a/rlcomposer/node.py b/rlcomposer/node.py index ecb03c8..7574638 100644 --- a/rlcomposer/node.py +++ b/rlcomposer/node.py @@ -182,7 +182,6 @@ def deserialize(self, data, hashmap={}): self.id = data["id"] hashmap[data["id"]] = self - self.param = data["param"] self.model_name = data["model_name"] @@ -203,6 +202,9 @@ def deserialize(self, data, hashmap={}): self.wrapper = ModelWrapper(self.nodeType) if self.model_name is not None: self.wrapper.loadModel(self.model_name) + if self.param is None: + self.param = self.wrapper.param + self.content.param_dict = self.param data["inputs"].sort(key=lambda socket: socket["index"] + socket["position"] * 100) data["outputs"].sort(key=lambda socket: socket["index"] + socket["position"] * 100) @@ -225,6 +227,4 @@ def deserialize(self, data, hashmap={}): self.scene.addNode(self) self.scene.grScene.addItem(self.grNode) - - return True diff --git a/rlcomposer/node_content.py b/rlcomposer/node_content.py index 329d93e..0421756 100644 --- a/rlcomposer/node_content.py +++ b/rlcomposer/node_content.py @@ -93,7 +93,7 @@ def __init__(self, content=None): self.button_clicked.connect(content.removeWindow) print(self.param) self.addWidgets() - print("After addwidgets",self.param) + print("After addwidgets", self.param) def addWidgets(self): @@ -129,7 +129,6 @@ def update(self): else: res[key] = obj.text() i += 1 - self.button_clicked.emit(res) self.close() diff --git a/rlcomposer/plot_button.py b/rlcomposer/plot_button.py new file mode 100644 index 0000000..5b2105e --- /dev/null +++ b/rlcomposer/plot_button.py @@ -0,0 +1,62 @@ +from PyQt5.QtWidgets import * +from PyQt5.QtCore import * + + +class PlotButton(QWidget): + def __init__(self, testing_reward, testing_action, testing_state, training_reward, training_action, parent=None): + super().__init__(parent) + self.testing_reward_widget = testing_reward + self.testing_action_widget = testing_action + self.testing_state_widget = testing_state + self.training_reward_widget = training_reward + self.training_action_widget = training_action + + self.initUI() + + def initUI(self): + layout = QGridLayout(self) + #layout.setRowStretch(0, 1) + #layout.setColumnStretch(0, 10) + #layout.setColumnStretch(1, 10) + #layout.setColumnStretch(2, 10) + self.setLayout(layout) + + self.training_plot_label = QLabel('Training Plots') + self.training_plot_label.setAlignment(Qt.AlignCenter) + self.testing_plot_label = QLabel('Testing Plots') + self.testing_plot_label.setAlignment(Qt.AlignCenter) + + self.training_reward_plot_button = QPushButton("Show Reward Plot", self) + self.training_reward_plot_button.clicked.connect(lambda: self.training_reward_widget.show()) + + self.training_action_plot_button = QPushButton("Show Action Plot", self) + self.training_action_plot_button.clicked.connect(lambda: self.training_action_widget.show()) + + self.testing_reward_plot_button = QPushButton("Show Reward Plot", self) + self.testing_reward_plot_button.clicked.connect(lambda: self.testing_reward_widget.show()) + + self.testing_state_plot_button = QPushButton("Show State Plot", self) + self.testing_state_plot_button.clicked.connect(lambda: self.testing_state_widget.show()) + + self.testing_action_plot_button = QPushButton("Show Action Plot", self) + self.testing_action_plot_button.clicked.connect(lambda: self.testing_action_widget.show()) + + layout.addWidget(self.training_plot_label, 0, 0, 1, 3) + sub_layout = QHBoxLayout() + sub_layout.addWidget(self.training_reward_plot_button) + sub_layout.addWidget(self.training_action_plot_button) + layout.addLayout(sub_layout, 1, 0, 1, 3) + + layout.addWidget(self.testing_plot_label, 2, 0, 1, 3) + layout.addWidget(self.testing_reward_plot_button, 3, 0) + layout.addWidget(self.testing_state_plot_button, 3, 1) + layout.addWidget(self.testing_action_plot_button, 3, 2) + + def set_testing_buttons(self, state_bool): + self.testing_reward_plot_button.setEnabled(state_bool) + self.testing_state_plot_button.setEnabled(state_bool) + self.testing_action_plot_button.setEnabled(state_bool) + + def set_training_buttons(self, state_bool): + self.training_reward_plot_button.setEnabled(state_bool) + self.training_action_plot_button.setEnabled(state_bool) diff --git a/rlcomposer/plot_widget.py b/rlcomposer/plot_widget.py index 977e55c..ef12f37 100644 --- a/rlcomposer/plot_widget.py +++ b/rlcomposer/plot_widget.py @@ -1,8 +1,10 @@ - -from PyQt5.QtWidgets import QWidget, QVBoxLayout, QSizePolicy - +from PyQt5.QtWidgets import QWidget, QVBoxLayout, QSizePolicy, QGridLayout, QTabWidget +from PyQt5.QtCore import * +from PyQt5.QtWidgets import * +from PyQt5.QtGui import * from matplotlib.backends.backend_qt5agg import NavigationToolbar2QT as NavigationToolbar import matplotlib +import pyqtgraph as pg matplotlib.use('Qt5Agg') @@ -10,37 +12,36 @@ from matplotlib.figure import Figure - - -class MplCanvas(FigureCanvas): - - def __init__(self, name, width=5, height=4, dpi=40): - - fig = Figure(figsize=(width, height), dpi=dpi) +class TestingMplCanvas(FigureCanvas): + def __init__(self, name, label, width=10, height=5, dpi=100): + fig = Figure(figsize=(width, height), dpi=dpi, constrained_layout=True) self.name = name self.axes = fig.add_subplot(111) - super(MplCanvas, self).__init__(fig) + super(TestingMplCanvas, self).__init__(fig) FigureCanvas.setSizePolicy(self, QSizePolicy.Expanding, QSizePolicy.Expanding) FigureCanvas.updateGeometry(self) - self.set_data() - - def set_data(self): + self.set_data(label) + def set_data(self, label): self.xdata = [] self.ydata = [] + self.label = label self.axes.cla() + self.axes.set_title(self.name) + self.axes.set_xlabel("Steps") - - self.update_names() - - def update_plot(self, step, value, label): + def update_list_data(self, step, value, label): self.xdata.append(step) self.ydata.append(value) - self.axes.cla() - self.update_names() - if len(label) == 1: - label = label[0] - self.axes.plot(self.xdata, self.ydata, label=label, marker="*") + self.label = label + + def refresh_plot(self): + self.axes.clear() + self.axes.set_title(self.name) + self.axes.set_xlabel("Steps") + if len(self.label) == 1: + self.label = self.label[0] + self.axes.plot(self.xdata, self.ydata, label=self.label, marker="*") self.axes.legend(loc='lower right') self.axes.grid() if len(self.ydata) > 100: @@ -49,15 +50,104 @@ def update_plot(self, step, value, label): self.axes.set_xlim(0, len(self.ydata)) self.draw() - def update_names(self): + +class TestingWidgetPlot(QWidget): + def __init__(self, name): + QWidget.__init__(self) + self.name = name + self.count = 0 + + self.layout = QGridLayout() + self.setLayout(self.layout) + self.setWindowTitle(self.name + " Plot") + + def set_canvas(self, n_envs, label): + self.plot_tab = QTabWidget() + self.canvas = [] + self.layout.addWidget(self.plot_tab, 0, 0) + for i in range(n_envs): + self.canvas.append(TestingMplCanvas(name=self.name, label=label)) + self.count += 1 + self.plot_tab.addTab(self.canvas[-1], f"Environment {self.count}") + + def clear_canvas(self): + self.canvas = [] + self.count = 0 + self.plot_tab.clear() + + def update_data(self, step, value, label): + for i in range(self.count): + self.canvas[i].update_list_data(step, value[i], label) + + def refresh(self): + for i in range(self.count): + self.canvas[i].refresh_plot() + + + + +class TrainingMplCanvas(FigureCanvas): + def __init__(self, name, label=None, width=10, height=5, dpi=100): + fig = Figure(figsize=(width, height), dpi=dpi, constrained_layout=True) + self.name = name + self.axes = fig.add_subplot(111) + super(TrainingMplCanvas, self).__init__(fig) + FigureCanvas.setSizePolicy(self, QSizePolicy.Expanding, QSizePolicy.Expanding) + FigureCanvas.updateGeometry(self) + self.set_data(label) + + def set_data(self, label): + self.xdata = [] + self.ydata = [] + self.label = label + self.axes.cla() self.axes.set_title(self.name) self.axes.set_xlabel("Steps") -class WidgetPlot(QWidget): + def update_list_data(self, step, value): + self.xdata.append(step) + self.ydata.append(value) + + def refresh_plot(self): + self.axes.clear() + self.axes.set_title(self.name) + self.axes.set_xlabel("Steps") + if len(self.label) == 1: + self.label = self.label[0] + self.axes.plot(self.xdata, self.ydata, label=self.label) + self.axes.legend(loc='lower right') + self.axes.grid() + self.draw() + + +class TrainingWidgetPlot(QWidget): def __init__(self, name): QWidget.__init__(self) - self.setLayout(QVBoxLayout()) - self.canvas = MplCanvas(name=name) - self.toolbar = NavigationToolbar(self.canvas, self) - self.layout().addWidget(self.toolbar) - self.layout().addWidget(self.canvas) \ No newline at end of file + self.name = name + self.count = 0 + + self.layout = QGridLayout() + self.setLayout(self.layout) + self.setWindowTitle(self.name + " Plot") + + def set_canvas(self, n_envs, label): + self.plot_tab = QTabWidget() + self.canvas = [] + self.layout.addWidget(self.plot_tab, 0, 0) + for i in range(n_envs): + self.canvas.append(TrainingMplCanvas(name=self.name, label=label)) + self.count += 1 + self.plot_tab.addTab(self.canvas[-1], f"Environment {self.count}") + + def clear_canvas(self): + self.canvas = [] + self.count = 0 + self.plot_tab.clear() + + def update_data(self, step, value): + for i in range(self.count): + self.canvas[i].update_list_data(step, value[i]) + + def refresh(self): + for i in range(self.count): + self.canvas[i].refresh_plot() \ No newline at end of file diff --git a/rlcomposer/rl/assets/delete.svg b/rlcomposer/rl/assets/delete.svg new file mode 100644 index 0000000..916644c --- /dev/null +++ b/rlcomposer/rl/assets/delete.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/rlcomposer/rl/assets/net.svg b/rlcomposer/rl/assets/net.svg new file mode 100644 index 0000000..3ecbbc5 --- /dev/null +++ b/rlcomposer/rl/assets/net.svg @@ -0,0 +1,37 @@ + + + + + + + + + + + + + +3x1x1 + + + + +3 + + + + +64 + + + + +1 +flatten + + +dense + + +dense + \ No newline at end of file diff --git a/rlcomposer/rl/assets/plus.svg b/rlcomposer/rl/assets/plus.svg new file mode 100644 index 0000000..26d5c2c --- /dev/null +++ b/rlcomposer/rl/assets/plus.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/rlcomposer/rl/assets/refresh.svg b/rlcomposer/rl/assets/refresh.svg new file mode 100644 index 0000000..1502e06 --- /dev/null +++ b/rlcomposer/rl/assets/refresh.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/rlcomposer/rl/components/__init__.py b/rlcomposer/rl/components/__init__.py new file mode 100644 index 0000000..11301d7 --- /dev/null +++ b/rlcomposer/rl/components/__init__.py @@ -0,0 +1,29 @@ +from gym.envs.registration import register + +register(id='Pendulum-v10', + entry_point='rlcomposer.rl.components.environments:Pendulum', + max_episode_steps=300) + +register(id='MountainCarEnv-v10', + entry_point='rlcomposer.rl.components.environments:MountainCarEnv', + max_episode_steps=300) + +register(id='Continuous_MountainCarEnv-v10', + entry_point='rlcomposer.rl.components.environments:Continuous_MountainCarEnv', + max_episode_steps=300) + +register(id='CartPoleEnv-v10', + entry_point='rlcomposer.rl.components.environments:CartPoleEnv', + max_episode_steps=300) + +register(id='AcrobotEnv-v10', + entry_point='rlcomposer.rl.components.environments:AcrobotEnv', + max_episode_steps=300) + +register(id='LunarLander-v10', + entry_point='rlcomposer.rl.components.environments:LunarLander', + max_episode_steps=300) + +register(id='SokobanEnv-v10', + entry_point='rlcomposer.rl.components.environments:SokobanEnv', + max_episode_steps=300) \ No newline at end of file diff --git a/rlcomposer/rl/components/environments.py b/rlcomposer/rl/components/environments.py index 8c6fdaa..4aef09c 100644 --- a/rlcomposer/rl/components/environments.py +++ b/rlcomposer/rl/components/environments.py @@ -5,130 +5,154 @@ from os import path import sys + +def disable_view_window(): + from gym.envs.classic_control import rendering + org_constructor = rendering.Viewer.__init__ + + def constructor(self, *args, **kwargs): + org_constructor(self, *args, **kwargs) + self.window.set_visible(visible=False) + + rendering.Viewer.__init__ = constructor + + def return_classes(): - unwanted = ["EzPickle", "circleShape", "contactListener", - "edgeShape", "fixtureDef", "polygonShape", - "revoluteJointDef", "FrictionDetector", "ContactDetector", "Car"] - current_module = sys.modules[__name__] - class_names = [] - for key in dir(current_module): - if key in unwanted: continue - if isinstance(getattr(current_module, key), type): - class_names.append(key) - return class_names + unwanted = ["EzPickle", "circleShape", "contactListener", "edgeShape", "fixtureDef", "polygonShape", + "revoluteJointDef", "FrictionDetector", "ContactDetector"] + current_module = sys.modules[__name__] + class_names = [] + for key in dir(current_module): + if key in unwanted: continue + if isinstance(getattr(current_module, key), type): + class_names.append(key) + return class_names + DEBUG = False + class Pendulum(gym.Env): - metadata = { - 'render.modes': ['human', 'rgb_array'], - 'video.frames_per_second': 30 - } - - def __init__(self, reward=None): - self.max_speed = 8 - self.max_torque = 2. - self.dt = .05 - self.g = 10.0 - self.m = 1. - self.l = 1. - self.reward_fn = reward - - self.viewer = None - - high = np.array([1., 1., self.max_speed], dtype=np.float32) - self.action_space = spaces.Box( - low=-self.max_torque, - high=self.max_torque, shape=(1,), - dtype=np.float32 - ) - self.observation_space = spaces.Box( - low=-high, - high=high, - dtype=np.float32 - ) - - self.seed() - - def seed(self, seed=None): - self.np_random, seed = seeding.np_random(seed) - return [seed] - - def step(self, u): - th, thdot = self.state # th := theta - - g = self.g - m = self.m - l = self.l - dt = self.dt - - u = np.clip(u, -self.max_torque, self.max_torque)[0] - self.last_u = u # for rendering - - - costs = self.reward_fn.calculateReward(th, thdot, u) - - newthdot = thdot + (-3 * g / (2 * l) * np.sin(th + np.pi) + 3. / (m * l ** 2) * u) * dt - newth = th + newthdot * dt - newthdot = np.clip(newthdot, -self.max_speed, self.max_speed) - if DEBUG: print("Inside step 1") - self.state = np.array([newth, newthdot]) - return self._get_obs(), -costs, False, {} - - def reset(self): - high = np.array([np.pi, 1]) - print("Reset 1") - self.state = self.np_random.uniform(low=-high, high=high) - print("Reset 2") - self.last_u = None - print(self.state) - return self._get_obs() - - def _get_obs(self): - theta, thetadot = self.state - print("Get obs 1") - return np.array([np.cos(theta), np.sin(theta), thetadot]) - - def render(self, mode='human'): - if self.viewer is None: - from gym.envs.classic_control import rendering - self.viewer = rendering.Viewer(500, 500) - self.viewer.set_bounds(-2.2, 2.2, -2.2, 2.2) - rod = rendering.make_capsule(1, .2) - rod.set_color(.8, .3, .3) - self.pole_transform = rendering.Transform() - rod.add_attr(self.pole_transform) - self.viewer.add_geom(rod) - axle = rendering.make_circle(.05) - axle.set_color(0, 0, 0) - self.viewer.add_geom(axle) - if DEBUG: print("Inside render 1") - try: - fname = path.join(path.dirname(__file__), "../assets/clockwise.png") - - if DEBUG: print("Inside render 2") - self.img = rendering.Image(fname, 1., 1.) - except Exception as e: - print(e) - if DEBUG: print("Inside render 3") - self.imgtrans = rendering.Transform() - if DEBUG: print("Inside render 4") - self.img.add_attr(self.imgtrans) - if DEBUG: print("Inside render 5") - self.viewer.add_onetime(self.img) - if DEBUG: print("Inside render 6") - self.pole_transform.set_rotation(self.state[0] + np.pi / 2) - if DEBUG: print("Inside render 7") - if self.last_u: - if DEBUG: print("Inside render 8") - self.imgtrans.scale = (-self.last_u / 2, np.abs(self.last_u) / 2) - if DEBUG: print("Inside render 9") - return self.viewer.render(return_rgb_array=mode == 'rgb_array') - - def close(self): - if self.viewer: - self.viewer.close() - self.viewer = None + metadata = { + 'render.modes': ['human', 'rgb_array'], + 'video.frames_per_second': 30 + } + + def __init__(self, reward=None): + self.max_speed = 8.0 + self.max_torque = 2.0 + self.dt = .05 + self.g = 10.0 + self.m = 1. + self.l = 1. + self.parameter_box = ['max_speed', 'max_torque', 'dt', 'g', 'm', 'l'] + self.reward_fn = reward + + self.viewer = None + + high = np.array([1., 1., self.max_speed], dtype=np.float32) + self.action_space = spaces.Box( + low=-self.max_torque, + high=self.max_torque, shape=(1,), + dtype=np.float32 + ) + self.observation_space = spaces.Box( + low=-high, + high=high, + dtype=np.float32 + ) + + self.viewer_x = 500 + self.viewer_y = 500 + self.seed() + + def seed(self, seed=None): + self.np_random, seed = seeding.np_random(seed) + return [seed] + + def step(self, u): + th, thdot = self.state # th := theta + + g = self.g + m = self.m + l = self.l + dt = self.dt + + u = np.clip(u, -self.max_torque, self.max_torque)[0] + self.last_u = u # for rendering + + costs = self.reward_fn.calculateReward(th, thdot, u) + + newthdot = thdot + (-3 * g / (2 * l) * np.sin(th + np.pi) + 3. / (m * l ** 2) * u) * dt + newth = th + newthdot * dt + newthdot = np.clip(newthdot, -self.max_speed, self.max_speed) + if DEBUG: print("Inside step 1") + self.state = np.array([newth, newthdot]) + return self._get_obs(), -costs, False, {} + + def reset(self): + disable_view_window() + high = np.array([np.pi, 1]) + print("Reset 1") + self.state = self.np_random.uniform(low=-high, high=high) + print("Reset 2") + self.last_u = None + print(self.state) + return self._get_obs() + + def set_render(self, n_envs): + if n_envs > 1 and n_envs <= 4: + self.viewer_x = int(self.viewer_x / 2) + self.viewer_y = int(self.viewer_y / 2) + elif n_envs > 4 and n_envs <= 9: + self.viewer_x = int(self.viewer_x / 3) + self.viewer_y = int(self.viewer_y / 3) + + def _get_obs(self): + theta, thetadot = self.state + print("Get obs 1") + return np.array([np.cos(theta), np.sin(theta), thetadot]) + + def render(self, mode='human'): + if self.viewer is None: + from gym.envs.classic_control import rendering + self.viewer = rendering.Viewer(self.viewer_x, self.viewer_y) + self.viewer.set_bounds(-2.2, 2.2, -2.2, 2.2) + rod = rendering.make_capsule(1, .2) + rod.set_color(.8, .3, .3) + self.pole_transform = rendering.Transform() + rod.add_attr(self.pole_transform) + self.viewer.add_geom(rod) + axle = rendering.make_circle(.05) + axle.set_color(0, 0, 0) + self.viewer.add_geom(axle) + if DEBUG: print("Inside render 1") + try: + fname = path.join(path.dirname(__file__), "../assets/clockwise.png") + + if DEBUG: print("Inside render 2") + self.img = rendering.Image(fname, 1., 1.) + except Exception as e: + print(e) + if DEBUG: print("Inside render 3") + self.imgtrans = rendering.Transform() + if DEBUG: print("Inside render 4") + self.img.add_attr(self.imgtrans) + if DEBUG: print("Inside render 5") + self.viewer.add_onetime(self.img) + if DEBUG: print("Inside render 6") + self.pole_transform.set_rotation(self.state[0] + np.pi / 2) + if DEBUG: print("Inside render 7") + if self.last_u: + if DEBUG: print("Inside render 8") + self.imgtrans.scale = (-self.last_u / 2, np.abs(self.last_u) / 2) + if DEBUG: print("Inside render 9") + return self.viewer.render(return_rgb_array=mode == 'rgb_array') + + def close(self): + if self.viewer: + self.viewer.close() + self.viewer = None @@ -157,10 +181,10 @@ def __init__(self, goal_velocity=0, reward=None): self.max_speed = 0.07 self.goal_position = 0.5 self.goal_velocity = goal_velocity - self.force = 0.001 self.gravity = 0.0025 + self.parameter_box = ['min_position', 'max_position', 'max_speed', 'goal_position', 'force', 'gravity'] self.reward_fn = reward self.low = np.array( @@ -177,6 +201,14 @@ def __init__(self, goal_velocity=0, reward=None): self.low, self.high, dtype=np.float32 ) + self.screen_width = 600 + self.screen_height = 400 + + self.world_width = self.max_position - self.min_position + self.scale = self.screen_width / self.world_width + self.carwidth = 40 + self.carheight = 20 + self.seed() def seed(self, seed=None): @@ -203,27 +235,40 @@ def step(self, action): return np.array(self.state), reward, done, {} def reset(self): + disable_view_window() self.state = np.array([self.np_random.uniform(low=-0.6, high=-0.4), 0]) return np.array(self.state) + def set_render(self, n_envs): + if n_envs > 1 and n_envs <= 4: + self.screen_width = int(self.screen_width / 2) + self.screen_height = int(self.screen_height / 2) + + self.world_width = self.max_position - self.min_position + self.scale = self.screen_width / self.world_width + self.carwidth = int(self.carwidth / 2) + self.carheight = int(self.carheight / 2) + elif n_envs > 4 and n_envs <= 9: + self.screen_width = int(self.screen_width / 3) + self.screen_height = int(self.screen_height / 3) + + self.world_width = self.max_position - self.min_position + self.scale = self.screen_width / self.world_width + self.carwidth = int(self.carwidth / 3) + self.carheight = int(self.carheight / 3) + def _height(self, xs): return np.sin(3 * xs) * .45 + .55 def render(self, mode='human'): - screen_width = 600 - screen_height = 400 - world_width = self.max_position - self.min_position - scale = screen_width / world_width - carwidth = 40 - carheight = 20 if self.viewer is None: from gym.envs.classic_control import rendering - self.viewer = rendering.Viewer(screen_width, screen_height) + self.viewer = rendering.Viewer(self.screen_width, self.screen_height) xs = np.linspace(self.min_position, self.max_position, 100) ys = self._height(xs) - xys = list(zip((xs - self.min_position) * scale, ys * scale)) + xys = list(zip((xs - self.min_position) * self.scale, ys * self.scale)) self.track = rendering.make_polyline(xys) self.track.set_linewidth(4) @@ -231,28 +276,28 @@ def render(self, mode='human'): clearance = 10 - l, r, t, b = -carwidth / 2, carwidth / 2, carheight, 0 + l, r, t, b = -self.carwidth / 2, self.carwidth / 2, self.carheight, 0 car = rendering.FilledPolygon([(l, b), (l, t), (r, t), (r, b)]) car.add_attr(rendering.Transform(translation=(0, clearance))) self.cartrans = rendering.Transform() car.add_attr(self.cartrans) self.viewer.add_geom(car) - frontwheel = rendering.make_circle(carheight / 2.5) + frontwheel = rendering.make_circle(self.carheight / 2.5) frontwheel.set_color(.5, .5, .5) frontwheel.add_attr( - rendering.Transform(translation=(carwidth / 4, clearance)) + rendering.Transform(translation=(self.carwidth / 4, clearance)) ) frontwheel.add_attr(self.cartrans) self.viewer.add_geom(frontwheel) - backwheel = rendering.make_circle(carheight / 2.5) + backwheel = rendering.make_circle(self.carheight / 2.5) backwheel.add_attr( - rendering.Transform(translation=(-carwidth / 4, clearance)) + rendering.Transform(translation=(-self.carwidth / 4, clearance)) ) backwheel.add_attr(self.cartrans) backwheel.set_color(.5, .5, .5) self.viewer.add_geom(backwheel) - flagx = (self.goal_position-self.min_position) * scale - flagy1 = self._height(self.goal_position) * scale + flagx = (self.goal_position-self.min_position) * self.scale + flagy1 = self._height(self.goal_position) * self.scale flagy2 = flagy1 + 50 flagpole = rendering.Line((flagx, flagy1), (flagx, flagy2)) self.viewer.add_geom(flagpole) @@ -264,7 +309,7 @@ def render(self, mode='human'): pos = self.state[0] self.cartrans.set_translation( - (pos-self.min_position) * scale, self._height(pos) * scale + (pos-self.min_position) * self.scale, self._height(pos) * self.scale ) self.cartrans.set_rotation(math.cos(3 * pos)) @@ -296,6 +341,9 @@ def __init__(self, goal_velocity=0, reward=None): self.goal_position = 0.45 # was 0.5 in gym, 0.45 in Arnaud de Broissia's version self.goal_velocity = goal_velocity self.power = 0.0015 + + self.parameter_box = ['min_action', 'max_action', 'min_position', 'max_position', 'max_speed', 'goal_position', + 'power'] self.reward_fn = reward @@ -320,6 +368,14 @@ def __init__(self, goal_velocity=0, reward=None): dtype=np.float32 ) + self.screen_width = 600 + self.screen_height = 400 + + self.world_width = self.max_position - self.min_position + self.scale = self.screen_width/self.world_width + self.carwidth = 40 + self.carheight = 20 + self.seed() self.reset() @@ -355,27 +411,39 @@ def step(self, action): return self.state, reward, done, {} def reset(self): + disable_view_window() self.state = np.array([self.np_random.uniform(low=-0.6, high=-0.4), 0]) return np.array(self.state) + def set_render(self, n_envs): + if n_envs > 1 and n_envs <= 4: + self.screen_width = int(self.screen_width / 2) + self.screen_height = int(self.screen_height / 2) + + self.world_width = self.max_position - self.min_position + self.scale = self.screen_width / self.world_width + self.carwidth = int(self.carwidth / 2) + self.carheight = int(self.carheight / 2) + elif n_envs > 4 and n_envs <= 9: + self.screen_width = int(self.screen_width / 3) + self.screen_height = int(self.screen_height / 3) + + self.world_width = self.max_position - self.min_position + self.scale = self.screen_width / self.world_width + self.carwidth = int(self.carwidth / 3) + self.carheight = int(self.carheight / 3) + def _height(self, xs): return np.sin(3 * xs)*.45+.55 def render(self, mode='human'): - screen_width = 600 - screen_height = 400 - - world_width = self.max_position - self.min_position - scale = screen_width/world_width - carwidth = 40 - carheight = 20 if self.viewer is None: from gym.envs.classic_control import rendering - self.viewer = rendering.Viewer(screen_width, screen_height) + self.viewer = rendering.Viewer(self.screen_width, self.screen_height) xs = np.linspace(self.min_position, self.max_position, 100) ys = self._height(xs) - xys = list(zip((xs-self.min_position)*scale, ys*scale)) + xys = list(zip((xs-self.min_position)*self.scale, ys*self.scale)) self.track = rendering.make_polyline(xys) self.track.set_linewidth(4) @@ -383,28 +451,28 @@ def render(self, mode='human'): clearance = 10 - l, r, t, b = -carwidth / 2, carwidth / 2, carheight, 0 + l, r, t, b = -self.carwidth / 2, self.carwidth / 2, self.carheight, 0 car = rendering.FilledPolygon([(l, b), (l, t), (r, t), (r, b)]) car.add_attr(rendering.Transform(translation=(0, clearance))) self.cartrans = rendering.Transform() car.add_attr(self.cartrans) self.viewer.add_geom(car) - frontwheel = rendering.make_circle(carheight / 2.5) + frontwheel = rendering.make_circle(self.carheight / 2.5) frontwheel.set_color(.5, .5, .5) frontwheel.add_attr( - rendering.Transform(translation=(carwidth / 4, clearance)) + rendering.Transform(translation=(self.carwidth / 4, clearance)) ) frontwheel.add_attr(self.cartrans) self.viewer.add_geom(frontwheel) - backwheel = rendering.make_circle(carheight / 2.5) + backwheel = rendering.make_circle(self.carheight / 2.5) backwheel.add_attr( - rendering.Transform(translation=(-carwidth / 4, clearance)) + rendering.Transform(translation=(-self.carwidth / 4, clearance)) ) backwheel.add_attr(self.cartrans) backwheel.set_color(.5, .5, .5) self.viewer.add_geom(backwheel) - flagx = (self.goal_position-self.min_position)*scale - flagy1 = self._height(self.goal_position)*scale + flagx = (self.goal_position-self.min_position)*self.scale + flagy1 = self._height(self.goal_position)*self.scale flagy2 = flagy1 + 50 flagpole = rendering.Line((flagx, flagy1), (flagx, flagy2)) self.viewer.add_geom(flagpole) @@ -416,7 +484,7 @@ def render(self, mode='human'): pos = self.state[0] self.cartrans.set_translation( - (pos-self.min_position) * scale, self._height(pos) * scale + (pos-self.min_position) * self.scale, self._height(pos) * self.scale ) self.cartrans.set_rotation(math.cos(3 * pos)) @@ -493,9 +561,9 @@ def __init__(self, reward=None): self.gravity = 9.8 self.masscart = 1.0 self.masspole = 0.1 - self.total_mass = (self.masspole + self.masscart) + self.total_mass = self.masspole + self.masscart self.length = 0.5 # actually half the pole's length - self.polemass_length = (self.masspole * self.length) + self.polemass_length = self.masspole * self.length self.force_mag = 10.0 self.tau = 0.02 # seconds between state updates self.kinematics_integrator = 'euler' @@ -504,6 +572,7 @@ def __init__(self, reward=None): self.theta_threshold_radians = 12 * 2 * math.pi / 360 self.x_threshold = 2.4 + self.parameter_box = ['gravity', 'masscart', 'masspole', 'length', 'force_mag'] # Angle limit set to 2 * theta_threshold_radians so failing observation # is still within bounds. high = np.array([self.x_threshold * 2, @@ -515,10 +584,20 @@ def __init__(self, reward=None): self.action_space = spaces.Discrete(2) self.observation_space = spaces.Box(-high, high, dtype=np.float32) + self.screen_width = 600 + self.screen_height = 400 + + self.world_width = self.x_threshold * 2 + self.scale = self.screen_width / self.world_width + self.carty = 100 # TOP OF CART + self.polewidth = 10.0 + self.polelen = self.scale * (2 * self.length) + self.cartwidth = 50.0 + self.cartheight = 30.0 + self.seed() self.viewer = None self.state = None - self.steps_beyond_done = None def seed(self, seed=None): @@ -582,44 +661,59 @@ def step(self, action): return np.array(self.state), reward, done, {} def reset(self): + disable_view_window() self.state = self.np_random.uniform(low=-0.05, high=0.05, size=(4,)) self.steps_beyond_done = None return np.array(self.state) - def render(self, mode='human'): - screen_width = 600 - screen_height = 400 + def set_render(self, n_envs): + if n_envs > 1 and n_envs <= 4: + self.screen_width = int(self.screen_width / 2) + self.screen_height = int(self.screen_height / 2) + + self.world_width = self.x_threshold * 2 + self.scale = self.screen_width / self.world_width + self.carty = int(self.carty / 2) + self.polewidth = int(self.polewidth / 2) + self.polelen = self.scale * (2 * self.length) + self.cartwidth = int(self.cartwidth / 2) + self.cartheight = int(self.cartheight / 2) + elif n_envs > 4 and n_envs <= 9: + self.screen_width = int(self.screen_width / 3) + self.screen_height = int(self.screen_height / 3) + + self.world_width = self.x_threshold * 2 + self.scale = self.screen_width / self.world_width + self.carty = int(self.carty / 3) + self.polewidth = int(self.polewidth / 3) + self.polelen = self.scale * (2 * self.length) + self.cartwidth = int(self.cartwidth / 3) + self.cartheight = int(self.cartheight / 3) - world_width = self.x_threshold * 2 - scale = screen_width/world_width - carty = 100 # TOP OF CART - polewidth = 10.0 - polelen = scale * (2 * self.length) - cartwidth = 50.0 - cartheight = 30.0 + def render(self, mode='human'): if self.viewer is None: from gym.envs.classic_control import rendering - self.viewer = rendering.Viewer(screen_width, screen_height) - l, r, t, b = -cartwidth / 2, cartwidth / 2, cartheight / 2, -cartheight / 2 - axleoffset = cartheight / 4.0 + self.viewer = rendering.Viewer(self.screen_width, self.screen_height) + l, r, t, b = -self.cartwidth / 2, self.cartwidth / 2, self.cartheight / 2, -self.cartheight / 2 + axleoffset = self.cartheight / 4.0 cart = rendering.FilledPolygon([(l, b), (l, t), (r, t), (r, b)]) self.carttrans = rendering.Transform() cart.add_attr(self.carttrans) self.viewer.add_geom(cart) - l, r, t, b = -polewidth / 2, polewidth / 2, polelen - polewidth / 2, -polewidth / 2 + l, r, t, b = -self.polewidth / 2, self.polewidth / 2, self.polelen - self.polewidth / 2, -self.polewidth / 2 pole = rendering.FilledPolygon([(l, b), (l, t), (r, t), (r, b)]) pole.set_color(.8, .6, .4) self.poletrans = rendering.Transform(translation=(0, axleoffset)) pole.add_attr(self.poletrans) pole.add_attr(self.carttrans) self.viewer.add_geom(pole) - self.axle = rendering.make_circle(polewidth/2) + self.axle = rendering.make_circle(self.polewidth/2) self.axle.add_attr(self.poletrans) self.axle.add_attr(self.carttrans) self.axle.set_color(.5, .5, .8) self.viewer.add_geom(self.axle) - self.track = rendering.Line((0, carty), (screen_width, carty)) + self.track = rendering.Line((0, self.carty), (self.screen_width, self.carty)) self.track.set_color(0, 0, 0) self.viewer.add_geom(self.track) @@ -630,12 +724,12 @@ def render(self, mode='human'): # Edit the pole polygon vertex pole = self._pole_geom - l, r, t, b = -polewidth / 2, polewidth / 2, polelen - polewidth / 2, -polewidth / 2 + l, r, t, b = -self.polewidth / 2, self.polewidth / 2, self.polelen - self.polewidth / 2, -self.polewidth / 2 pole.v = [(l, b), (l, t), (r, t), (r, b)] x = self.state - cartx = x[0] * scale + screen_width / 2.0 # MIDDLE OF CART - self.carttrans.set_translation(cartx, carty) + cartx = x[0] * self.scale + self.screen_width / 2.0 # MIDDLE OF CART + self.carttrans.set_translation(cartx, self.carty) self.poletrans.set_rotation(-x[2]) return self.viewer.render(return_rgb_array=mode == 'rgb_array') @@ -740,7 +834,21 @@ def __init__(self, reward=None): self.observation_space = spaces.Box(low=low, high=high, dtype=np.float32) self.action_space = spaces.Discrete(3) self.state = None + + self.m1 = self.LINK_MASS_1 + self.m2 = self.LINK_MASS_2 + self.l1 = self.LINK_LENGTH_1 + self.lc1 = self.LINK_COM_POS_1 + self.lc2 = self.LINK_COM_POS_2 + self.I1 = self.LINK_MOI + self.I2 = self.LINK_MOI + self.g = 9.8 + + self.viewer_x = 500 + self.viewer_y = 500 + self.reward_fn = reward + self.parameter_box = ['m1', 'm2', 'l1', 'lc1', 'lc2', 'I1', 'I2', 'g'] self.seed() def seed(self, seed=None): @@ -748,6 +856,7 @@ def seed(self, seed=None): return [seed] def reset(self): + disable_view_window() self.state = self.np_random.uniform(low=-0.1, high=0.1, size=(4,)) return self._get_ob() @@ -791,47 +900,47 @@ def _terminal(self): return bool(-cos(s[0]) - cos(s[1] + s[0]) > 1.) def _dsdt(self, s_augmented, t): - m1 = self.LINK_MASS_1 - m2 = self.LINK_MASS_2 - l1 = self.LINK_LENGTH_1 - lc1 = self.LINK_COM_POS_1 - lc2 = self.LINK_COM_POS_2 - I1 = self.LINK_MOI - I2 = self.LINK_MOI - g = 9.8 a = s_augmented[-1] s = s_augmented[:-1] theta1 = s[0] theta2 = s[1] dtheta1 = s[2] dtheta2 = s[3] - d1 = m1 * lc1 ** 2 + m2 * \ - (l1 ** 2 + lc2 ** 2 + 2 * l1 * lc2 * cos(theta2)) + I1 + I2 - d2 = m2 * (lc2 ** 2 + l1 * lc2 * cos(theta2)) + I2 - phi2 = m2 * lc2 * g * cos(theta1 + theta2 - pi / 2.) - phi1 = - m2 * l1 * lc2 * dtheta2 ** 2 * sin(theta2) \ - - 2 * m2 * l1 * lc2 * dtheta2 * dtheta1 * sin(theta2) \ - + (m1 * lc1 + m2 * l1) * g * cos(theta1 - pi / 2) + phi2 + d1 = self.m1 * self.lc1 ** 2 + self.m2 * \ + (self.l1 ** 2 + self.lc2 ** 2 + 2 * self.l1 * self.lc2 * cos(theta2)) + self.I1 + self.I2 + d2 = self.m2 * (self.lc2 ** 2 + self.l1 * self.lc2 * cos(theta2)) + self.I2 + phi2 = self.m2 * self.lc2 * self.g * cos(theta1 + theta2 - pi / 2.) + phi1 = - self.m2 * self.l1 * self.lc2 * dtheta2 ** 2 * sin(theta2) \ + - 2 * self.m2 * self.l1 * self.lc2 * dtheta2 * dtheta1 * sin(theta2) \ + + (self.m1 * self.lc1 + self.m2 * self.l1) * self.g * cos(theta1 - pi / 2) + phi2 if self.book_or_nips == "nips": # the following line is consistent with the description in the # paper ddtheta2 = (a + d2 / d1 * phi1 - phi2) / \ - (m2 * lc2 ** 2 + I2 - d2 ** 2 / d1) + (self.m2 * self.lc2 ** 2 + self.I2 - d2 ** 2 / d1) else: # the following line is consistent with the java implementation and the # book - ddtheta2 = (a + d2 / d1 * phi1 - m2 * l1 * lc2 * dtheta1 ** 2 * sin(theta2) - phi2) \ - / (m2 * lc2 ** 2 + I2 - d2 ** 2 / d1) + ddtheta2 = (a + d2 / d1 * phi1 - self.m2 * self.l1 * self.lc2 * dtheta1 ** 2 * sin(theta2) - phi2) \ + / (self.m2 * self.lc2 ** 2 + self.I2 - d2 ** 2 / d1) ddtheta1 = -(d2 * ddtheta2 + phi1) / d1 return (dtheta1, dtheta2, ddtheta1, ddtheta2, 0.) + def set_render(self, n_envs): + if n_envs > 1 and n_envs <= 4: + self.viewer_x = int(self.viewer_x / 2) + self.viewer_y = int(self.viewer_y / 2) + elif n_envs > 4 and n_envs <= 9: + self.viewer_x = int(self.viewer_x / 3) + self.viewer_y = int(self.viewer_y / 3) + def render(self, mode='human'): from gym.envs.classic_control import rendering s = self.state if self.viewer is None: - self.viewer = rendering.Viewer(500,500) + self.viewer = rendering.Viewer(self.viewer_x,self.viewer_y) bound = self.LINK_LENGTH_1 + self.LINK_LENGTH_2 + 0.2 # 2.2 for default self.viewer.set_bounds(-bound,bound,-bound,bound) @@ -959,12 +1068,14 @@ def derivs(x,t): -""" + import sys, math import numpy as np import Box2D -from Box2D.b2 import edgeShape, circleShape, fixtureDef, polygonShape, revoluteJointDef, contactListener +#from Box2D.b2 import (edgeShape, circleShape, fixtureDef, polygonShape, revoluteJointDef, contactListener) +from Box2D.Box2D import b2EdgeShape as edgeShape, b2CircleShape as circleShape, b2FixtureDef as fixtureDef, \ + b2PolygonShape as polygonShape, b2RevoluteJointDef as revoluteJointDef, b2ContactListener as contactListener import gym from gym import spaces @@ -979,7 +1090,7 @@ def derivs(x,t): INITIAL_RANDOM = 1000.0 # Set 1500 to make game harder LANDER_POLY =[ - (-14, +17), (-17, 0), (-17 ,-10), + (-14, +17), (-17, 0), (-17, -10), (+17, -10), (+17, 0), (+14, +17) ] LEG_AWAY = 20 @@ -1022,9 +1133,9 @@ class LunarLander(gym.Env, EzPickle): def __init__(self, reward=None): EzPickle.__init__(self) - self.seed() self.viewer = None self.reward_fn = reward + self.seed() self.world = Box2D.b2World() self.moon = None @@ -1032,6 +1143,10 @@ def __init__(self, reward=None): self.particles = [] self.prev_reward = None + self.parameter_box = [] + + self.VIEWPORT_W = 600 + self.VIEWPORT_H = 400 # useful range is -1 .. +1, but spikes can be higher self.observation_space = spaces.Box(-np.inf, np.inf, shape=(8,), dtype=np.float32) @@ -1062,15 +1177,25 @@ def _destroy(self): self.world.DestroyBody(self.legs[0]) self.world.DestroyBody(self.legs[1]) + def set_render(self, n_envs): + print("AAAA", n_envs) + if n_envs > 1 and n_envs <= 4: + self.VIEWPORT_W = int(self.VIEWPORT_W / 2) + self.VIEWPORT_H = int(self.VIEWPORT_H / 2) + elif n_envs > 4 and n_envs <= 9: + self.VIEWPORT_W = int(self.VIEWPORT_W / 3) + self.VIEWPORT_H = int(self.VIEWPORT_H / 3) + def reset(self): + disable_view_window() self._destroy() self.world.contactListener_keepref = ContactDetector(self) self.world.contactListener = self.world.contactListener_keepref self.game_over = False self.prev_shaping = None - W = VIEWPORT_W/SCALE - H = VIEWPORT_H/SCALE + W = self.VIEWPORT_W/SCALE + H = self.VIEWPORT_H/SCALE # terrain CHUNKS = 11 @@ -1100,9 +1225,9 @@ def reset(self): self.moon.color1 = (0.0, 0.0, 0.0) self.moon.color2 = (0.0, 0.0, 0.0) - initial_y = VIEWPORT_H/SCALE + initial_y = self.VIEWPORT_H/SCALE self.lander = self.world.CreateDynamicBody( - position=(VIEWPORT_W/SCALE/2, initial_y), + position=(self.VIEWPORT_W/SCALE/2, initial_y), angle=0.0, fixtures = fixtureDef( shape=polygonShape(vertices=[(x/SCALE, y/SCALE) for x, y in LANDER_POLY]), @@ -1122,7 +1247,7 @@ def reset(self): self.legs = [] for i in [-1, +1]: leg = self.world.CreateDynamicBody( - position=(VIEWPORT_W/SCALE/2 - i*LEG_AWAY/SCALE, initial_y), + position=(self.VIEWPORT_W/SCALE/2 - i*LEG_AWAY/SCALE, initial_y), angle=(i * 0.05), fixtures=fixtureDef( shape=polygonShape(box=(LEG_W/SCALE, LEG_H/SCALE)), @@ -1239,10 +1364,10 @@ def step(self, action): pos = self.lander.position vel = self.lander.linearVelocity state = [ - (pos.x - VIEWPORT_W/SCALE/2) / (VIEWPORT_W/SCALE/2), - (pos.y - (self.helipad_y+LEG_DOWN/SCALE)) / (VIEWPORT_H/SCALE/2), - vel.x*(VIEWPORT_W/SCALE/2)/FPS, - vel.y*(VIEWPORT_H/SCALE/2)/FPS, + (pos.x - self.VIEWPORT_W/SCALE/2) / (self.VIEWPORT_W/SCALE/2), + (pos.y - (self.helipad_y+LEG_DOWN/SCALE)) / (self.VIEWPORT_H/SCALE/2), + vel.x*(self.VIEWPORT_W/SCALE/2)/FPS, + vel.y*(self.VIEWPORT_H/SCALE/2)/FPS, self.lander.angle, 20.0*self.lander.angularVelocity/FPS, 1.0 if self.legs[0].ground_contact else 0.0, @@ -1257,15 +1382,14 @@ def step(self, action): - 100*abs(state[4]) + 10*state[6] + 10*state[7] # And ten points for legs contact, the idea is if you # lose contact again after landing, you get negative reward if self.prev_shaping is not None: - # reward = shaping - self.prev_shaping - reward = self.reward_fn.prevShaping(shaping, self.prev_shaping) + reward = shaping - self.prev_shaping + # reward = self.reward_fn.prevShaping(shaping, self.prev_shaping) self.prev_shaping = shaping + # reward = self.reward_fn.fuelCalculate(reward, m_power, s_power) - reward = self.reward_fn.fuelCalculate(reward, m_power, s_power) - - # reward -= m_power*0.30 # less fuel spent is better, about -30 for heuristic landing - # reward -= s_power*0.03 + reward -= m_power*0.30 # less fuel spent is better, about -30 for heuristic landing + reward -= s_power*0.03 done = False if self.game_over or abs(state[0]) >= 1.0: @@ -1280,8 +1404,8 @@ def step(self, action): def render(self, mode='human'): from gym.envs.classic_control import rendering if self.viewer is None: - self.viewer = rendering.Viewer(VIEWPORT_W, VIEWPORT_H) - self.viewer.set_bounds(0, VIEWPORT_W/SCALE, 0, VIEWPORT_H/SCALE) + self.viewer = rendering.Viewer(self.VIEWPORT_W, self.VIEWPORT_H) + self.viewer.set_bounds(0, self.VIEWPORT_W/SCALE, 0, self.VIEWPORT_H/SCALE) for obj in self.particles: obj.ttl -= 0.15 @@ -1320,4 +1444,314 @@ def close(self): self.viewer.close() self.viewer = None -""" \ No newline at end of file + + + + + + + + + + + + +import gym +from gym.utils import seeding +from gym import spaces +from gym_sokoban.envs.room_utils import generate_room +from gym_sokoban.envs.render_utils import room_to_rgb, room_to_tiny_world_rgb +import numpy as np + + +class SokobanEnv(gym.Env): + metadata = { + 'render.modes': ['human', 'rgb_array', 'tiny_human', 'tiny_rgb_array', 'raw'] + } + + def __init__(self, + dim_room=(10, 10), + max_steps=120, + num_boxes=4, + num_gen_steps=None, + reset=False): + + # General Configuration + self.dim_room = dim_room + if num_gen_steps == None: + self.num_gen_steps = int(1.7 * (dim_room[0] + dim_room[1])) + else: + self.num_gen_steps = num_gen_steps + + self.num_boxes = num_boxes + self.boxes_on_target = 0 + + # Penalties and Rewards + self.penalty_for_step = -0.1 + self.penalty_box_off_target = -1 + self.reward_box_on_target = 1 + self.reward_finished = 10 + self.reward_last = 0 + + # Other Settings + self.viewer = None + self.max_steps = max_steps + self.action_space = spaces.Discrete(len(ACTION_LOOKUP)) + screen_height, screen_width = (dim_room[0] * 16, dim_room[1] * 16) + self.observation_space = spaces.Box(low=0, high=255, shape=(screen_height, screen_width, 3), dtype=np.uint8) + self.reward_fn = None + self.parameter_box = ['num_boxes', 'max_steps', 'penalty_for_step', 'penalty_box_off_target', + 'reward_box_on_target', 'reward_finished'] + + if reset: + # Initialize Room + _ = self.reset() + + def seed(self, seed=None): + self.np_random, seed = seeding.np_random(seed) + return [seed] + + def step(self, action, observation_mode='rgb_array'): + assert action in ACTION_LOOKUP + assert observation_mode in ['rgb_array', 'tiny_rgb_array', 'raw'] + + self.num_env_steps += 1 + + self.new_box_position = None + self.old_box_position = None + + moved_box = False + + if action == 0: + moved_player = False + + # All push actions are in the range of [0, 3] + elif action < 5: + moved_player, moved_box = self._push(action) + + else: + moved_player = self._move(action) + + self._calc_reward() + + done = self._check_if_done() + + # Convert the observation to RGB frame + observation = self.render(mode=observation_mode) + + info = { + "action.name": ACTION_LOOKUP[action], + "action.moved_player": moved_player, + "action.moved_box": moved_box, + } + if done: + info["maxsteps_used"] = self._check_if_maxsteps() + info["all_boxes_on_target"] = self._check_if_all_boxes_on_target() + + return observation, self.reward_last, done, info + + def _push(self, action): + """ + Perform a push, if a box is adjacent in the right direction. + If no box, can be pushed, try to move. + :param action: + :return: Boolean, indicating a change of the room's state + """ + change = CHANGE_COORDINATES[(action - 1) % 4] + new_position = self.player_position + change + current_position = self.player_position.copy() + + # No push, if the push would get the box out of the room's grid + new_box_position = new_position + change + if new_box_position[0] >= self.room_state.shape[0] \ + or new_box_position[1] >= self.room_state.shape[1]: + return False, False + + can_push_box = self.room_state[new_position[0], new_position[1]] in [3, 4] + can_push_box &= self.room_state[new_box_position[0], new_box_position[1]] in [1, 2] + if can_push_box: + + self.new_box_position = tuple(new_box_position) + self.old_box_position = tuple(new_position) + + # Move Player + self.player_position = new_position + self.room_state[(new_position[0], new_position[1])] = 5 + self.room_state[current_position[0], current_position[1]] = \ + self.room_fixed[current_position[0], current_position[1]] + + # Move Box + box_type = 4 + if self.room_fixed[new_box_position[0], new_box_position[1]] == 2: + box_type = 3 + self.room_state[new_box_position[0], new_box_position[1]] = box_type + return True, True + + # Try to move if no box to push, available + else: + return self._move(action), False + + def _move(self, action): + """ + Moves the player to the next field, if it is not occupied. + :param action: + :return: Boolean, indicating a change of the room's state + """ + change = CHANGE_COORDINATES[(action - 1) % 4] + new_position = self.player_position + change + current_position = self.player_position.copy() + + # Move player if the field in the moving direction is either + # an empty field or an empty box target. + if self.room_state[new_position[0], new_position[1]] in [1, 2]: + self.player_position = new_position + self.room_state[(new_position[0], new_position[1])] = 5 + self.room_state[current_position[0], current_position[1]] = \ + self.room_fixed[current_position[0], current_position[1]] + + return True + + return False + + def _calc_reward(self): + """ + Calculate Reward Based on + :return: + """ + # Every step a small penalty is given, This ensures + # that short solutions have a higher reward. + self.reward_last = self.penalty_for_step + + # count boxes off or on the target + empty_targets = self.room_state == 2 + player_on_target = (self.room_fixed == 2) & (self.room_state == 5) + total_targets = empty_targets | player_on_target + + current_boxes_on_target = self.num_boxes - \ + np.where(total_targets)[0].shape[0] + + # Add the reward if a box is pushed on the target and give a + # penalty if a box is pushed off the target. + if current_boxes_on_target > self.boxes_on_target: + self.reward_last += self.reward_box_on_target + elif current_boxes_on_target < self.boxes_on_target: + self.reward_last += self.penalty_box_off_target + + game_won = self._check_if_all_boxes_on_target() + if game_won: + self.reward_last += self.reward_finished + + self.boxes_on_target = current_boxes_on_target + + def _check_if_done(self): + # Check if the game is over either through reaching the maximum number + # of available steps or by pushing all boxes on the targets. + return self._check_if_all_boxes_on_target() or self._check_if_maxsteps() + + def _check_if_all_boxes_on_target(self): + empty_targets = self.room_state == 2 + player_hiding_target = (self.room_fixed == 2) & (self.room_state == 5) + are_all_boxes_on_targets = np.where(empty_targets | player_hiding_target)[0].shape[0] == 0 + return are_all_boxes_on_targets + + def _check_if_maxsteps(self): + return (self.max_steps == self.num_env_steps) + + def reset(self, second_player=False, render_mode='rgb_array'): + try: + self.room_fixed, self.room_state, self.box_mapping = generate_room( + dim=self.dim_room, + num_steps=self.num_gen_steps, + num_boxes=self.num_boxes, + second_player=second_player + ) + except (RuntimeError, RuntimeWarning) as e: + print("[SOKOBAN] Runtime Error/Warning: {}".format(e)) + print("[SOKOBAN] Retry . . .") + return self.reset(second_player=second_player, render_mode=render_mode) + + self.player_position = np.argwhere(self.room_state == 5)[0] + self.num_env_steps = 0 + self.reward_last = 0 + self.boxes_on_target = 0 + + starting_observation = self.render(render_mode) + return starting_observation + + def set_render(self, n_envs): + pass + + def render(self, mode='rgb_array', close=None, scale=1): + assert mode in RENDERING_MODES + + img = self.get_image(mode, scale) + + if 'rgb_array' in mode: + return img + + elif 'human' in mode: + from gym.envs.classic_control import rendering + if self.viewer is None: + self.viewer = rendering.SimpleImageViewer() + self.viewer.imshow(img) + return self.viewer.isopen + + elif 'raw' in mode: + arr_walls = (self.room_fixed == 0).view(np.int8) + arr_goals = (self.room_fixed == 2).view(np.int8) + arr_boxes = ((self.room_state == 4) + (self.room_state == 3)).view(np.int8) + arr_player = (self.room_state == 5).view(np.int8) + + return arr_walls, arr_goals, arr_boxes, arr_player + + else: + super(SokobanEnv, self).render(mode=mode) # just raise an exception + + def get_image(self, mode, scale=1): + + if mode.startswith('tiny_'): + img = room_to_tiny_world_rgb(self.room_state, self.room_fixed, scale=scale) + else: + img = room_to_rgb(self.room_state, self.room_fixed) + + return img + + def close(self): + if self.viewer is not None: + self.viewer.close() + + def set_maxsteps(self, num_steps): + self.max_steps = num_steps + + def get_action_lookup(self): + return ACTION_LOOKUP + + def get_action_meanings(self): + return ACTION_LOOKUP + + +ACTION_LOOKUP = { + 0: 'no operation', + 1: 'push up', + 2: 'push down', + 3: 'push left', + 4: 'push right', + 5: 'move up', + 6: 'move down', + 7: 'move left', + 8: 'move right', +} + +# Moves are mapped to coordinate changes as follows +# 0: Move up +# 1: Move down +# 2: Move left +# 3: Move right +CHANGE_COORDINATES = { + 0: (-1, 0), + 1: (1, 0), + 2: (0, -1), + 3: (0, 1) +} + +RENDERING_MODES = ['rgb_array', 'human', 'tiny_rgb_array', 'tiny_human', 'raw'] diff --git a/rlcomposer/rl/components/models.py b/rlcomposer/rl/components/models.py index 0b7dbaa..25b8ac9 100644 --- a/rlcomposer/rl/components/models.py +++ b/rlcomposer/rl/components/models.py @@ -1,4 +1,5 @@ import sys +from stable_baselines3 import * def return_classes(): current_module = sys.modules[__name__] @@ -6,6 +7,7 @@ def return_classes(): for key in dir(current_module): if isinstance(getattr(current_module, key), type): class_names.append(key) + class_names.remove("HER") return class_names @@ -24,6 +26,5 @@ class A2C(): class DDPG(): pass - class TD3(): pass diff --git a/rlcomposer/rl/components/rewards.py b/rlcomposer/rl/components/rewards.py index 379bc51..cc4736c 100644 --- a/rlcomposer/rl/components/rewards.py +++ b/rlcomposer/rl/components/rewards.py @@ -10,15 +10,15 @@ def return_classes(): class_names.append(key) return class_names -### Reward Function for Pendulum environment ### -def angle_normalize(x): - return (((x + np.pi) % (2 * np.pi)) - np.pi) class PendulumReward(): def __init__(self): pass def calculateReward(self, th, thdot, u): + def angle_normalize(x): + return (((x + np.pi) % (2 * np.pi)) - np.pi) + return angle_normalize(th) ** 2 + .1 * thdot ** 2 + .001 * (u ** 2) ############################################################################### @@ -80,10 +80,6 @@ class LunarReward(): def __init__(self): pass - def calculateReward(self, terminal): - reward = -1. if not terminal else 0. - return reward - def prevShaping(self, a, b): return a-b @@ -92,3 +88,11 @@ def fuelCalculate(self, reward, m_power, s_power): reward -= s_power*0.03 return reward + + +class SokobanReward(): + def __init__(self): + pass + + def calculateReward(self): + pass diff --git a/rlcomposer/rl/env_wrapper.py b/rlcomposer/rl/env_wrapper.py index e26265a..699dc5f 100644 --- a/rlcomposer/rl/env_wrapper.py +++ b/rlcomposer/rl/env_wrapper.py @@ -1,34 +1,57 @@ import sys import importlib +import os +from stable_baselines3.common.vec_env.subproc_vec_env import SubprocVecEnv +from stable_baselines3.common.utils import set_random_seed +from stable_baselines3.common.monitor import Monitor +import gym def excludeVariables(obj): - dic = vars(obj) + dic = vars(obj.env.env) res_dic = {} - for (key, val) in dic.items(): - if (type(val) == float or type(val) == int): - res_dic[key] = val + parameter_box = dic['parameter_box'] + for key in parameter_box: + res_dic[key] = dic[key] + # for (key, val) in dic.items(): + # if (type(val) == float or type(val) == int): + # res_dic[key] = val + return res_dic +def make_env(env_id, rank, seed=None): + if isinstance(env_id, str): + env = gym.make(env_id) + else: + env = env_id() + if seed is not None: + env.seed(seed + rank) + env.action_space.seed(seed + rank) + env = Monitor(env, filename=None) + return env + class EnvWrapper(): def __init__(self, env_name=""): self.env_name = env_name - self.setEnv() - # self.setParameters(self.param) + self.env = None + self.setEnv(0) + self.setParameters(self.param) - def setEnv(self): - from components.environments import Pendulum - module = importlib.import_module("components.environments") + def setEnv(self, rank): + module = importlib.import_module(".environments", "rlcomposer.rl.components") print(sys.modules[__name__]) - self.env = getattr(module, self.env_name)() + #self.env = make_vec_env(self.env_name+'-v10', n_envs=1, seed=set_random_seed(0), vec_env_cls=SubprocVecEnv) + self.env = make_env(self.env_name+'-v10', rank) self.param = excludeVariables(self.env) + def callable_env(self): + def _init(): + return self.env + return _init def setReward(self, reward): - setattr(self.env, "reward_fn", reward) + setattr(self.env.env.env, 'reward_fn', reward) def setParameters(self, param): - for key,value in param.items(): - setattr(self.env, key, value) - - + for key, value in param.items(): + setattr(self.env.env.env, key, value) diff --git a/rlcomposer/rl/instance.py b/rlcomposer/rl/instance.py index 037fc65..9406c01 100644 --- a/rlcomposer/rl/instance.py +++ b/rlcomposer/rl/instance.py @@ -1,59 +1,67 @@ from stable_baselines3 import * +import stable_baselines3.common.logger as logger +from stable_baselines3.common.utils import get_latest_run_id +from stable_baselines3.common.vec_env.subproc_vec_env import SubprocVecEnv +from stable_baselines3.common.vec_env import VecFrameStack from .tensorboard_callbacks import Callback -import sys, subprocess +import sys, subprocess, webbrowser, os from tensorboard import program - DEBUG = False + def disable_view_window(): - from gym.envs.classic_control import rendering - org_constructor = rendering.Viewer.__init__ + from gym.envs.classic_control import rendering + org_constructor = rendering.Viewer.__init__ - def constructor(self, *args, **kwargs): - org_constructor(self, *args, **kwargs) - self.window.set_visible(visible=False) + def constructor(self, *args, **kwargs): + org_constructor(self, *args, **kwargs) + self.window.set_visible(visible=False) - rendering.Viewer.__init__ = constructor + rendering.Viewer.__init__ = constructor class Instance(): def __init__(self, scene): self.scene = scene - self.env_wrapper, self.reward_wrapper, self.model_wrapper = None, None, None + self.env_wrapper_list, self.reward_wrapper, self.model_wrapper = [], None, None self.env = None self.model = None + self.reward_func = None self.tensorboard_log = None + self.logger = logger self.buildInstance() - def buildInstance(self): disable_view_window() - current_env = None - for item in self.scene.nodes: - if item.title == "Environment": - current_env = item.wrapper.env for item in self.scene.nodes: if item.title == "Environment": - self.env_wrapper = item.wrapper + self.env_wrapper_list.append(item.wrapper) elif item.title == "Reward": self.reward_wrapper = item.wrapper elif item.title == "Models": self.model_wrapper = item.wrapper - self.model_wrapper.setModel(current_env) + self.reward_func = self.reward_wrapper.reward - self.env_wrapper.setReward(self.reward_func) - self.env = self.env_wrapper.env + for env_wrapper in self.env_wrapper_list: + env_wrapper.setReward(self.reward_func) + self.env = SubprocVecEnv([env_wrapper.callable_env() for env_wrapper in self.env_wrapper_list]) + self.model_wrapper.setModel(self.env) + self.tensorboard_log = self.env_wrapper_list[0].env_name + "_" + self.model_wrapper.model_name + setattr(self.model_wrapper.model, "tensorboard_log", self.tensorboard_log) self.model = self.model_wrapper.model - self.tensorboard_log = self.env_wrapper.env_name + "_" + self.model_wrapper.model_name - print(self.model) + if self.scene.model_archive is not None: + self.model = self.scene.model_archive - - def train_model(self): + def train_model(self, network, signal, plots): + self.model_wrapper.add_parameters(network, self.tensorboard_log) self.model = self.model_wrapper.model - setattr(self.model, "tensorboard_log", self.tensorboard_log) - self.model.learn(self.model_wrapper.total_timesteps, callback=Callback()) + print(getattr(self.model, "policy_kwargs")) + self.tensorboard(browser=False, folder=str(self.model_wrapper.model_name + "_" + str(get_latest_run_id(self.tensorboard_log, self.model_wrapper.model_name)+1))) + signal.url.emit(self.url) + self.model.learn(total_timesteps=self.model_wrapper.total_timesteps, callback=Callback(self.tensorboard_log, signal, plots)) + self.scene.model_archive = self.model def step(self): action, _ = self.model.predict(self.state) @@ -65,8 +73,10 @@ def step(self): def prep(self): if DEBUG: print(self.env) + print(len(self.env_wrapper_list)) self.state = self.env.reset() if DEBUG: print("resetted") + self.env.env_method('set_render', len(self.env_wrapper_list)) img = self.env.render(mode="rgb_array") if DEBUG: print(type(img)) return img @@ -77,25 +87,30 @@ def save(self, filename): def removeInstance(self): pass - def tensorboard(self, browser=True): + def tensorboard(self, browser=True, folder = None): # Kill current session self._tensorboard_kill() + print('New session of tensorboard.') # Open the dir of the current env - if sys.platform == 'win32': + print(self.tensorboard_log) + self.url = 'Null' + if False: # sys.platform == 'win32': tensorboard.program cannot close manually try: tb = program.TensorBoard() - tb.configure(argv=[None, '--logdir', self.tensorboard_log]) - url = tb.launch() + tb.configure(argv=[None, '--logdir', self.tensorboard_log+"/"+folder]) + self.url = tb.launch() cmd = '' - except: + except Exception as e: + print("Tensorboard Error:", e) pass else: - cmd = 'tensorboard --logdir {} --port 6006'.format(self.tensorboard_log) #--reload_interval 1 + cmd = 'tensorboard --logdir {} --reload_multifile true --reload_interval 2'.format(self.tensorboard_log+"/"+folder) # --reload_interval 1 try: DEVNULL = open(os.devnull, 'wb') subprocess.Popen(cmd, shell=True, stdout=DEVNULL, stderr=DEVNULL) except: + print('Tensorboard Error') pass diff --git a/rlcomposer/rl/model_wrapper.py b/rlcomposer/rl/model_wrapper.py index 894501d..8f7d1db 100644 --- a/rlcomposer/rl/model_wrapper.py +++ b/rlcomposer/rl/model_wrapper.py @@ -1,126 +1,136 @@ import sys from stable_baselines3 import * + class ModelWrapper(): - def __init__(self, model_name=""): - self.model = None - self.env = None - self.model_name = model_name - self.initParam() + def __init__(self, model_name=""): + self.model = None + self.env = None + self.model_name = model_name + self.initParam() + + def initParam(self): + self.param = {} + if self.model_name == "DQN": + self.param = {"total_timesteps": 20000, + "policy": "MlpPolicy", + "learning_rate": 0.0001, + "buffer_size": 1000000, + "learning_starts": 50000, + "batch_size": 32, + "tau": 1.0, + "gamma": 0.99, + "train_freq": 4, + "gradient_steps": 1, + "target_update_interval": 10000, + "exploration_fraction": 0.1, + "exploration_initial_eps": 1.0, + "exploration_final_eps": 0.05, + "max_grad_norm": 10, + } + elif self.model_name == "SAC": + self.param = {"total_timesteps": 20000, + "policy": "MlpPolicy", + "learning_rate": 1e-3, + "buffer_size": 1000000, + "batch_size": 256, + "tau": 0.005, + "gamma": 0.99, + "learning_starts": 100, + "train_freq": 1, + "gradient_steps": 1 + } - def initParam(self): - self.param = {} - if self.model_name == "DQN": - self.param = {"total_timesteps": 20000, - "policy": "MlpPolicy", - "learning_rate": 0.0001, - "buffer_size": 1000000, - "learning_starts": 50000, - "batch_size": 32, - "tau": 1.0, - "gamma": 0.99, - "train_freq": 4, - "gradient_steps": 1, - "target_update_interval": 10000, - "exploration_fraction": 0.1, - "exploration_initial_eps": 1.0, - "exploration_final_eps": 0.05, - "max_grad_norm": 10, - } + elif self.model_name == "A2C": + self.param = {"total_timesteps": 20000, + "policy": "MlpPolicy", + "learning_rate": 0.0007, + "n_steps": 5, + "gamma": 0.99, + "gae_lambda": 1.0, + "ent_coef": 0.0, + "vf_coef": 0.5, + "max_grad_norm": 0.5, + "rms_prop_eps": 1e-05 + } - elif self.model_name == "SAC": - self.param = {"total_timesteps": 20000, - "policy": "MlpPolicy", - "learning_rate": 1e-3, - "buffer_size": 1000000, - "batch_size": 256, - "tau": 0.005, - "gamma": 0.99, - "learning_starts": 100, - "train_freq": 1, - "gradient_steps": 1 - } + elif self.model_name == "DDPG": + self.param = {"total_timesteps": 20000, + "policy": "MlpPolicy", + "learning_rate": 0.001, + "buffer_size": 1000000, + "learning_starts": 100, + "batch_size": 100, + "tau": 0.005, + "gamma": 0.99, + } + elif self.model_name == "PPO": + self.param = {"total_timesteps": 20000, + "policy": "MlpPolicy", + "learning_rate": 0.0003, + "n_steps": 2048, + "batch_size": 64, + "n_epochs": 10, + "gae_lambda": 0.95, + "gamma": 0.99, + "clip_range": 0.2, + "ent_coef": 0.0, + "vf_coef": 0.5, + "max_grad_norm": 0.5, + } - elif self.model_name == "A2C": - self.param = {"total_timesteps": 20000, - "policy": "MlpPolicy", - "learning_rate": 0.0007, - "n_steps": 5, - "gamma": 0.99, - "gae_lambda": 1.0, - "ent_coef": 0.0, - "vf_coef": 0.5, - "max_grad_norm": 0.5, - "rms_prop_eps": 1e-05 - } + elif self.model_name == "TD3": + self.param = {"total_timesteps": 10000, + "policy": "MlpPolicy", + "learning_rate": 0.001, + "buffer_size": 1000000, + "learning_starts": 100, + "batch_size": 100, + "tau": 0.005, + "gamma": 0.99, + "policy_delay": 2, + "target_policy_noise": 0.2, + "target_noise_clip": 0.5 + } + try: + self.total_timesteps = self.param["total_timesteps"] + except: + pass - elif self.model_name == "DDPG": - self.param = {"total_timesteps": 10000, - "policy": "MlpPolicy", - "learning_rate":0.001, - "buffer_size": 1000000, - "learning_starts": 100, - "batch_size": 100, - "tau": 0.005, - "gamma": 0.99, - } - elif self.model_name == "PPO": - self.param = {"total_timesteps": 20000, - "policy": "MlpPolicy", - "learning_rate": 0.0003, - "n_steps": 2048, - "batch_size": 64, - "n_epochs": 10, - "gae_lambda": 0.95, - "gamma": 0.99, - "clip_range": 0.2, - "ent_coef": 0.0, - "vf_coef": 0.5, - "max_grad_norm": 0.5, - } + def setModel(self, env): + self.env = env + if self.model is None: + self.model = getattr(sys.modules[__name__], self.model_name)( + env=self.env, + **without(self.param, "total_timesteps")) - elif self.model_name == "TD3": - self.param = {"total_timesteps": 10000, - "policy": "MlpPolicy", - "learning_rate": 0.001, - "buffer_size": 1000000, - "learning_starts": 100, - "batch_size": 100, - "tau": 0.005, - "gamma": 0.99, - "policy_delay": 2, - "target_policy_noise": 0.2, - "target_noise_clip": 0.5 - } - try: - self.total_timesteps = self.param["total_timesteps"] - except: - pass + def loadModel(self, dir): + model_str = dir.split('/')[-1].split('_')[0] + self.model = getattr(sys.modules[__name__], model_str).load(dir.split("/")[-1].split(".")[0]) - def setModel(self,env): - self.env = env - if self.model is None: - self.model = getattr(sys.modules[__name__], self.model_name)( - env=self.env, - **without(self.param, "total_timesteps")) + def setParameters(self, param): + for key, value in param.items(): + if key == "total_timesteps": + continue + self.param[key] = value + self.total_timesteps = param["total_timesteps"] + if not self.env is None: + self.model = getattr(sys.modules[__name__], self.model_name)( + env=self.env, + **without(self.param, "total_timesteps")) - def loadModel(self, dir): - model_str = dir.split('/')[-1].split('_')[0] - self.model = getattr(sys.modules[__name__], model_str).load(dir.split("/")[-1].split(".")[0]) + def add_parameters(self, network, log): + param_copy = self.param.copy() + if network["enabled"]: + param_copy["policy_kwargs"] = network["conf"].copy() + param_copy["tensorboard_log"] = log + self.model = getattr(sys.modules[__name__], self.model_name)( + env=self.env, + **without(param_copy, "total_timesteps")) - def setParameters(self, param): - for key,value in param.items(): - if key == "total_timesteps": - continue - self.param[key] = value - self.total_timesteps = param["total_timesteps"] - if not self.env is None: - self.model = getattr(sys.modules[__name__], self.model_name)( - env=self.env, - **without(self.param, "total_timesteps")) def without(d, key): - new_d = d.copy() - new_d.pop(key) - return new_d \ No newline at end of file + new_d = d.copy() + new_d.pop(key) + return new_d diff --git a/rlcomposer/rl/tensorboard_callbacks.py b/rlcomposer/rl/tensorboard_callbacks.py index 0e92fa1..3d6db16 100644 --- a/rlcomposer/rl/tensorboard_callbacks.py +++ b/rlcomposer/rl/tensorboard_callbacks.py @@ -1,19 +1,61 @@ from stable_baselines3.common.callbacks import BaseCallback -from stable_baselines3.common.logger import TensorBoardOutputFormat +#from stable_baselines3.common.logger import TensorBoardOutputFormat +#from stable_baselines3.common.logger import Figure +#import tensorflow as tf +from stable_baselines3.common.logger import Logger + class Callback(BaseCallback): - def __init__(self): - super().__init__() + def __init__(self, tensorboard_log, signal, plots): + super(Callback, self).__init__() self.states = [] self.actions = [] + self.values_dict = dict({'A2C': ['state/']}) + self.plots = plots + self.signal = signal + self.signal.progress.emit(0) + self.tensorboard_log = tensorboard_log def _on_step(self): - self.states.append(self.locals["obs_tensor"].detach().cpu().numpy()) - self.actions.append(self.locals["actions"]) - self.logger.record('actions', self.locals["actions"][0]) - for i in range (self.states[0].shape[1]): - self.logger.record('states_' + str(i), self.locals["obs_tensor"][0,i]) - return True + self.signal.progress.emit(100 - int(100*self.model._current_progress_remaining)) + + if self.locals['tb_log_name'] in ['A2C', 'PPO']: + for i in range(0, self.locals["new_obs"].shape[0]): + for j in range(0, self.locals["new_obs"].shape[1]): + self.logger.record(f'state/Environment {i+1}, State {j+1}', self.locals["new_obs"][i, j]) + self.logger.record(f'train/Environment {i+1}, rewards', self.locals["rewards"][i]) + self.logger.record(f'train/Environment {i+1}, values', self.locals["values"][i][0]) + self.logger.record(f'train/Environment {i+1}, log_probs', self.locals["log_probs"][i]) + if len(self.locals["actions"].shape) > 1: + self.logger.record(f'train/Environment {i + 1}, actions', self.locals["actions"][i][0]) + else: + self.logger.record(f'train/Environment {i + 1}, actions', self.locals["actions"][i]) + log_dict = self.logger.get_log_dict() + rewards, actions = [], [] + for i in range(0, self.locals["new_obs"].shape[0]): + rewards.append(log_dict[f'train/Environment {i+1}, rewards']) + actions.append(log_dict[f'train/Environment {i+1}, actions']) + self.plots[0].update_data(self.num_timesteps, rewards) + self.plots[1].update_data(self.num_timesteps, actions) + + if self.locals['tb_log_name'] in ['SAC', 'DQN', 'TD3', 'DDPG']: + for i in range(0, self.locals["new_obs"].shape[1]): + self.logger.record(f'state/State {i+1}', self.locals["new_obs"][0, i]) + self.logger.record('train/rewards', self.locals["reward"][0]) + #self.logger.record('train/actions', self.locals["action"][0][0]) + if len(self.locals["action"].shape) > 1: + self.logger.record('train/actions', self.locals["action"][0][0]) + else: + self.logger.record('train/actions', self.locals["action"][0]) + log_dict = self.logger.get_log_dict() + self.plots[0].update_data(self.num_timesteps, [log_dict['train/rewards']]) + self.plots[1].update_data(self.num_timesteps, [log_dict['train/actions']]) + + self.logger.dump(step=self.num_timesteps) + #for i in self.locals.keys(): + # print(i, self.locals[i]) + return self.signal.finished_value def _on_training_end(self): + self.signal.progress.emit(100 - int(100*self.model._current_progress_remaining)) pass diff --git a/rlcomposer/scene.py b/rlcomposer/scene.py index 176baa1..a9013a2 100644 --- a/rlcomposer/scene.py +++ b/rlcomposer/scene.py @@ -1,6 +1,10 @@ import json from collections import OrderedDict import random +import os +import xmltodict +import xml.etree.ElementTree as et +import json from .edge import Edge from .node import Node @@ -16,6 +20,7 @@ def __init__(self): self.nodes = [] self.edges = [] self.width, self.height = 3200, 3200 + self.model_archive = None self.initUI() self.history = SceneHistory(self) @@ -75,6 +80,65 @@ def loadFromFile(self, file): self.is_modified = False + def loadFromGraphML(self, filename): + xml_doc_path = os.path.abspath(filename) + xml_tree = et.parse(xml_doc_path) + root = xml_tree.getroot() + # set encoding to and method proper + to_string = et.tostring(root, encoding='UTF-8', method='xml') + xml_to_dict = xmltodict.parse(to_string) + print(json.dumps(xml_to_dict)) + data = {'nodes': [], 'edges': []} + + for edge in xml_to_dict['ns0:graphml']['ns0:graph']['ns0:edge']: + data['edges'].append(OrderedDict([ + ('id', edge['@id']), + ('start', edge['@source']), + ('end', edge['@target']) + ])) + + for node in xml_to_dict['ns0:graphml']['ns0:graph']['ns0:node']: + if 'Reward' in node['ns0:data']['ns2:ShapeNode']['ns2:NodeLabel']: + title = 'Reward' + elif len(node['ns0:data']['ns2:ShapeNode']['ns2:NodeLabel']) <= 4: + title = 'Models' + else: + title = 'Environment' + inputs, outputs = [], [] + for edge in data['edges']: + if edge['start'] == node['@id']: + outputs.append(OrderedDict([ + ('id', node['@id']), + ('index', 0), + ('position', 4), + ('is_input', 0) + ])) + if edge['end'] == node['@id']: + inputs.append(OrderedDict([ + ('id', node['@id']), + ('index', 0), + ('position', 1), + ('is_input', 1) + ])) + + temp_dict = OrderedDict([ + ('id', node['@id']), + ("title", title), + ("x_pos", float(node['ns0:data']['ns2:ShapeNode']['ns2:Geometry']['@x'])), + ("y_pos", float(node['ns0:data']['ns2:ShapeNode']['ns2:Geometry']['@y'])), + ("inputs", inputs), + ("outputs", outputs), + ("input_nodes", []), + ("output_nodes", []), + ("content", {}), + ("param", None), + ("nodeType", node['ns0:data']['ns2:ShapeNode']['ns2:NodeLabel']), + ("model_name", None), + ]) + data['nodes'].append(temp_dict) + + self.deserialize(data) + def clear(self): while len(self.nodes) > 0: self.nodes[0].remove() diff --git a/rlcomposer/tensorboard_widget.py b/rlcomposer/tensorboard_widget.py index 509b9ae..875c4d5 100644 --- a/rlcomposer/tensorboard_widget.py +++ b/rlcomposer/tensorboard_widget.py @@ -16,19 +16,34 @@ class Tensorboard(QWebEngineView): _update() Loads the URL that tensorboard is running """ + url_Signal = pyqtSignal(str) def __init__(self): super(Tensorboard, self).__init__() + self.url = 'http://localhost:6006/' # initialize the timer - self.timer = QTimer() - # it only works only once - self.timer.setSingleShot(True) - # timer is connected to _update() function - self.timer.timeout.connect(self._update) + # self.timer = QTimer() + # self.timer.setSingleShot(True) + # self.timer.timeout.connect(self._update) + + self.timer_initial = QTimer() + self.timer_initial.setSingleShot(True) + self.timer_initial.timeout.connect(self._update) + + def initial_load(self, delay_ms=3000): + self.timer_initial.start(delay_ms) + + #def delayed_load(self, delay_ms=10000): + # self.load(QUrl('http://localhost:6006/')) + # self.timer.start(delay_ms) + + def setURL(self, url): + delay_ms = 3000 + if url != 'Null': + self.url = url + self.timer_initial.start(delay_ms) - def delayed_load(self, delay_ms=2500): - # - self.timer.start(delay_ms) def _update(self): - self.load(QUrl('http://localhost:6006/#scalars&_smoothingWeight=0.99')) - self.setZoomFactor(0.6) \ No newline at end of file + self.load(QUrl(self.url)) + self.setZoomFactor(1.1) + self.timer_initial.stop() diff --git a/rlcomposer/test_plots.py b/rlcomposer/test_plots.py deleted file mode 100644 index bce72d8..0000000 --- a/rlcomposer/test_plots.py +++ /dev/null @@ -1,28 +0,0 @@ -from PyQt5.QtWidgets import * - -class TestPlots(QWidget): - def __init__(self, reward, action, state, parent=None): - super().__init__(parent) - self.raw_plot_widget = reward - self.state_plot_widget = state - self.action_plot_widget = action - self.initUI() - - - def initUI(self): - layout = QGridLayout(self) - layout.setRowStretch(0, 1) - - layout.setColumnStretch(0, 10) - layout.setColumnStretch(1, 10) - layout.setColumnStretch(2, 10) - - - self.setLayout(layout) - - layout.addWidget(self.raw_plot_widget) - layout.addWidget(self.action_plot_widget) - layout.addWidget(self.state_plot_widget) - layout.addWidget(self.raw_plot_widget, 0, 0) - layout.addWidget(self.state_plot_widget, 0, 1) - layout.addWidget(self.action_plot_widget, 0, 2) diff --git a/rlcomposer/treeview_widget.py b/rlcomposer/treeview_widget.py index b017396..0fd59f8 100644 --- a/rlcomposer/treeview_widget.py +++ b/rlcomposer/treeview_widget.py @@ -26,11 +26,11 @@ def __init__(self, txt='', font_size=12, set_bold=False, color=QColor(0, 0, 0)): class FunctionTree(QWidget): - def __init__(self, scene): + def __init__(self, window_widget): super().__init__() self.setWindowTitle('Node Function') self.layout = QGridLayout() - self.mainScene = scene + self.window_widget = window_widget self.treeView = QTreeView() self.treeView.setHeaderHidden(True) @@ -65,7 +65,7 @@ def initTreeModel(self): self.treeView.setModel(self.treeModel) self.treeView.expandAll() self.treeView.doubleClicked.connect(self.getValue) - self.layout.addWidget(self.treeView, 0,0,1,2) + self.layout.addWidget(self.treeView, 0, 0, 1, 10) self.layout.setContentsMargins(0, 0, 0, 0) self.addWidgets() self.setLayout(self.layout) @@ -76,38 +76,58 @@ def createItem(self, name): def addWidgets(self): self.widget = QWidget(self) - inpsocket = QLabel('Inputs') - outsocket = QLabel('Outputs') + inpsocket = QLabel('Inputs:') + outsocket = QLabel('Outputs:') - self.inpsocketEdit = QLineEdit() - self.outsocketEdit = QLineEdit() + self.status = QLabel('Status:') + self.progress_bar = QProgressBar() + self.progress_bar.setRange(0, 100) + self.progress_bar.setAlignment(Qt.AlignCenter) + self.progress_bar.setValue(0) + + self.inpsocketEdit = QSpinBox() + self.inpsocketEdit.setMinimum(0) + self.inpsocketEdit.setMaximum(4) + self.inpsocketEdit.setProperty('value', 0) + self.outsocketEdit = QSpinBox() + self.outsocketEdit.setMinimum(0) + self.outsocketEdit.setMaximum(4) + self.outsocketEdit.setProperty('value', 0) self.push = QPushButton("Create Node", self) self.push.clicked.connect(self.onButtonClick) - self.layout.addWidget(inpsocket, 1, 0) - self.layout.addWidget(self.inpsocketEdit, 1, 1) + self.layout.addWidget(self.inpsocketEdit, 1, 2) self.layout.addWidget(outsocket, 2, 0) - self.layout.addWidget(self.outsocketEdit, 2, 1) + self.layout.addWidget(self.outsocketEdit, 2, 2) + + self.layout.addWidget(self.status, 1, 6, 2, 1) - self.layout.addWidget(self.push, 3, 0) + self.layout.addWidget(self.push, 3, 0, 1, 3) + self.layout.addWidget(self.progress_bar, 3, 3, 1, 7) def getValue(self, val): print(val.data()) print(val.row()) print(val.column()) + def progress_bar_handler(self, value): + self.progress_bar.setValue(value) + @pyqtSlot() def onButtonClick(self): try: inpNum = int(self.inpsocketEdit.text()) outNum = int(self.outsocketEdit.text()) + index = self.treeView.selectedIndexes()[0] + except IndexError: + print("Select a Module!") + return except Exception as e: print("Fill the editable textboxes!") return - index = self.treeView.selectedIndexes()[0] nodeType = index.model().itemFromIndex(index).text() parentTitle = index.model().itemFromIndex(index.parent()).text() @@ -119,5 +139,6 @@ def onButtonClick(self): if os.path.isfile(fname): model_name = fname + self.mainScene = self.window_widget.get_scene() self.mainScene.generateNode(parentTitle, inpNum, outNum, nodeType=nodeType, model_name=model_name) self.mainScene.history.storeHistory("Created " + parentTitle + " by dock widget", setModified=True) diff --git a/rlcomposer/window_widget.py b/rlcomposer/window_widget.py index 505bb67..2a2c37b 100644 --- a/rlcomposer/window_widget.py +++ b/rlcomposer/window_widget.py @@ -1,4 +1,5 @@ from PyQt5.QtWidgets import * +from PyQt5.QtGui import QIcon from .scene import Scene from .graphics_view import QDMGraphicsView @@ -7,19 +8,38 @@ class RLComposerWindow(QWidget): def __init__(self, parent=None): super().__init__(parent) - + self.scene_list = [] + self.view_list = [] + self.view = None + self.scene = None self.initUI() def initUI(self): - self.setGeometry(200, 200, 800, 600) self.layout = QVBoxLayout() self.layout.setContentsMargins(0, 0, 0, 0) self.setLayout(self.layout) + self.scene_tab = QTabWidget() + self.layout.addWidget(self.scene_tab) + self.scene_tab.currentChanged.connect(self.onTabChange) - # create graphics scene - self.scene = Scene() + self.tabButton = QToolButton(self) + self.tabButton.setIcon(QIcon('rlcomposer/rl/assets/plus.svg')) + self.tabButton.clicked.connect(self.add_page) + + self.scene_tab.setCornerWidget(self.tabButton) + self.add_page() + def add_page(self): + # create graphics scene + self.scene_list.append(Scene()) # create graphic view - self.view = QDMGraphicsView(self.scene.grScene, self) - self.view.setScene(self.scene.grScene) - self.layout.addWidget(self.view) + self.view_list.append(QDMGraphicsView(self.scene_list[-1].grScene, self)) + self.view_list[-1].setScene(self.scene_list[-1].grScene) + self.scene_tab.addTab(self.view_list[-1], f"Scene {self.scene_tab.count()+1}") + + def onTabChange(self, i): + self.scene = self.scene_list[i] + self.view = self.view_list[i] + + def get_scene(self): + return self.scene