diff --git a/11 Quantum ML/作业HW12/HW12_EN.pdf b/11 Quantum ML/作业HW12/HW12_EN.pdf new file mode 100644 index 0000000..f07f555 Binary files /dev/null and b/11 Quantum ML/作业HW12/HW12_EN.pdf differ diff --git a/11 Quantum ML/作业HW12/hw12_reinforcement_learning_chinese_version.ipynb b/11 Quantum ML/作业HW12/hw12_reinforcement_learning_chinese_version.ipynb new file mode 100644 index 0000000..771a9aa --- /dev/null +++ b/11 Quantum ML/作业HW12/hw12_reinforcement_learning_chinese_version.ipynb @@ -0,0 +1,3645 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "hw12_reinforcement_learning_chinese_version.ipynb", + "provenance": [], + "collapsed_sections": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "2acab9542fe64b979fa2ac2adb3f10a8": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_f288c64b5ff748eb82178bf1de17934f", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_de34e5b178f5470e98e0275102a65042", + "IPY_MODEL_c93cba301cac439ca56fb6b45bd1c4e4" + ] + } + }, + "f288c64b5ff748eb82178bf1de17934f": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "de34e5b178f5470e98e0275102a65042": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_43c6ee720b674626ab3a869bda5dd6e3", + "_dom_classes": [], + "description": "Total: -24.0, Final: -40.0: 100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 400, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 400, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_2465d2b109d34922a486341232d86ad6" + } + }, + "c93cba301cac439ca56fb6b45bd1c4e4": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_aa27187195be4da9874025395eac35eb", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 400/400 [11:02<00:00, 1.66s/it]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_02d196d4f9734f998455d92bd9300adb" + } + }, + "43c6ee720b674626ab3a869bda5dd6e3": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "2465d2b109d34922a486341232d86ad6": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "aa27187195be4da9874025395eac35eb": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "02d196d4f9734f998455d92bd9300adb": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + } + } + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Fp30SB4bxeQb" + }, + "source": [ + "# **Homework 12 - Reinforcement Learning**\n", + "\n", + "若有任何問題，歡迎來信至助教信箱 ntu-ml-2021spring-ta@googlegroups.com\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yXsnCWPtWSNk" + }, + "source": [ + "## 前置作業\n", + "\n", + "首先我們需要安裝必要的系統套件及 PyPi 套件。\n", + "gym 這個套件由 OpenAI 所提供，是一套用來開發與比較 Reinforcement Learning 演算法的工具包（toolkit）。\n", + "而其餘套件則是為了在 Notebook 中繪圖所需要的套件。" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "5e2bScpnkVbv", + "outputId": "dd8cf053-de15-4a11-c146-5f3405d1e377" + }, + "source": [ + "!apt update\n", + "!apt install python-opengl xvfb -y\n", + "!pip install gym[box2d]==0.18.3 pyvirtualdisplay tqdm numpy==1.19.5 torch==1.8.1" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "\u001b[33m\r0% [Working]\u001b[0m\r \rGet:1 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease [3,626 B]\n", + "\u001b[33m\r0% [Connecting to archive.ubuntu.com (91.189.88.142)] [Waiting for headers] [1 \u001b[0m\u001b[33m\r0% [Connecting to archive.ubuntu.com (91.189.88.142)] [Waiting for headers] [Co\u001b[0m\r \rIgn:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 InRelease\n", + "\u001b[33m\r0% [Connecting to archive.ubuntu.com (91.189.88.142)] [Waiting for headers] [Co\u001b[0m\u001b[33m\r0% [1 InRelease gpgv 3,626 B] [Connecting to archive.ubuntu.com (91.189.88.142)\u001b[0m\r \rIgn:3 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 InRelease\n", + "\u001b[33m\r0% [1 InRelease gpgv 3,626 B] [Connecting to archive.ubuntu.com (91.189.88.142)\u001b[0m\r \rGet:4 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 Release [697 B]\n", + "\u001b[33m\r0% [1 InRelease gpgv 3,626 B] [Connecting to archive.ubuntu.com (91.189.88.142)\u001b[0m\u001b[33m\r0% [1 InRelease gpgv 3,626 B] [Connecting to archive.ubuntu.com (91.189.88.142)\u001b[0m\r \rHit:5 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 Release\n", + "\u001b[33m\r0% [1 InRelease gpgv 3,626 B] [Connecting to archive.ubuntu.com (91.189.88.142)\u001b[0m\r \rGet:6 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 Release.gpg [836 B]\n", + "Get:7 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n", + "Hit:8 http://archive.ubuntu.com/ubuntu bionic InRelease\n", + "Get:9 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease [15.9 kB]\n", + "Get:10 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ Packages [60.9 kB]\n", + "Get:11 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n", + "Hit:13 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n", + "Ign:14 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 Packages\n", + "Get:14 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 Packages [798 kB]\n", + "Get:15 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n", + "Hit:16 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease\n", + "Get:17 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease [21.3 kB]\n", + "Get:18 http://security.ubuntu.com/ubuntu bionic-security/restricted amd64 Packages [423 kB]\n", + "Get:19 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [2,152 kB]\n", + "Get:20 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main Sources [1,770 kB]\n", + "Get:21 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [1,413 kB]\n", + "Get:22 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [2,184 kB]\n", + "Get:23 http://archive.ubuntu.com/ubuntu bionic-updates/restricted amd64 Packages [452 kB]\n", + "Get:24 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [2,584 kB]\n", + "Get:25 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main amd64 Packages [905 kB]\n", + "Get:26 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic/main amd64 Packages [41.5 kB]\n", + "Fetched 13.1 MB in 4s (3,031 kB/s)\n", + "Reading package lists... Done\n", + "Building dependency tree \n", + "Reading state information... Done\n", + "86 packages can be upgraded. Run 'apt list --upgradable' to see them.\n", + "Reading package lists... Done\n", + "Building dependency tree \n", + "Reading state information... Done\n", + "The following package was automatically installed and is no longer required:\n", + " libnvidia-common-460\n", + "Use 'apt autoremove' to remove it.\n", + "Suggested packages:\n", + " libgle3\n", + "The following NEW packages will be installed:\n", + " python-opengl xvfb\n", + "0 upgraded, 2 newly installed, 0 to remove and 86 not upgraded.\n", + "Need to get 1,281 kB of archives.\n", + "After this operation, 7,686 kB of additional disk space will be used.\n", + "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python-opengl all 3.1.0+dfsg-1 [496 kB]\n", + "Get:2 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 xvfb amd64 2:1.19.6-1ubuntu4.9 [784 kB]\n", + "Fetched 1,281 kB in 1s (977 kB/s)\n", + "Selecting previously unselected package python-opengl.\n", + "(Reading database ... 160706 files and directories currently installed.)\n", + "Preparing to unpack .../python-opengl_3.1.0+dfsg-1_all.deb ...\n", + "Unpacking python-opengl (3.1.0+dfsg-1) ...\n", + "Selecting previously unselected package xvfb.\n", + "Preparing to unpack .../xvfb_2%3a1.19.6-1ubuntu4.9_amd64.deb ...\n", + "Unpacking xvfb (2:1.19.6-1ubuntu4.9) ...\n", + "Setting up python-opengl (3.1.0+dfsg-1) ...\n", + "Setting up xvfb (2:1.19.6-1ubuntu4.9) ...\n", + "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n", + "Requirement already satisfied: gym[box2d] in /usr/local/lib/python3.7/dist-packages (0.17.3)\n", + "Collecting pyvirtualdisplay\n", + " Downloading https://files.pythonhosted.org/packages/19/88/7a198a5ee3baa3d547f5a49574cd8c3913b216f5276b690b028f89ffb325/PyVirtualDisplay-2.1-py3-none-any.whl\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (4.41.1)\n", + "Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.3.0)\n", + "Requirement already satisfied: numpy>=1.10.4 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.19.5)\n", + "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.5.0)\n", + "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.4.1)\n", + "Collecting box2d-py~=2.3.5; extra == \"box2d\"\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/87/34/da5393985c3ff9a76351df6127c275dcb5749ae0abbe8d5210f06d97405d/box2d_py-2.3.8-cp37-cp37m-manylinux1_x86_64.whl (448kB)\n", + "\u001b[K |████████████████████████████████| 450kB 10.3MB/s \n", + "\u001b[?25hCollecting EasyProcess\n", + " Downloading https://files.pythonhosted.org/packages/48/3c/75573613641c90c6d094059ac28adb748560d99bd27ee6f80cce398f404e/EasyProcess-0.3-py2.py3-none-any.whl\n", + "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym[box2d]) (0.16.0)\n", + "Installing collected packages: EasyProcess, pyvirtualdisplay, box2d-py\n", + "Successfully installed EasyProcess-0.3 box2d-py-2.3.8 pyvirtualdisplay-2.1\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "M_-i3cdoYsks" + }, + "source": [ + "接下來，設置好 virtual display，並引入所有必要的套件。" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "nl2nREINDLiw" + }, + "source": [ + "%%capture\n", + "from pyvirtualdisplay import Display\n", + "virtual_display = Display(visible=0, size=(1400, 900))\n", + "virtual_display.start()\n", + "\n", + "%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from IPython import display\n", + "\n", + "import numpy as np\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.optim as optim\n", + "import torch.nn.functional as F\n", + "from torch.distributions import Categorical\n", + "from tqdm.notebook import tqdm" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HVu9-Vdrl4E3" + }, + "source": [ + "# 請不要更改 random seed !!!!\n", + "# 不然在judgeboi上 你的成績不會被reproduce !!!!" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "fV9i8i2YkRbO" + }, + "source": [ + "seed = 543 # Do not change this\n", + "def fix(env, seed):\n", + " env.seed(seed)\n", + " env.action_space.seed(seed)\n", + " torch.manual_seed(seed)\n", + " torch.cuda.manual_seed(seed)\n", + " torch.cuda.manual_seed_all(seed)\n", + " np.random.seed(seed)\n", + " random.seed(seed)\n", + " torch.set_deterministic(True)\n", + " torch.backends.cudnn.benchmark = False\n", + " torch.backends.cudnn.deterministic = True" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "He0XDx6bzjgC" + }, + "source": [ + "最後，引入 OpenAI 的 gym，並建立一個 [Lunar Lander](https://gym.openai.com/envs/LunarLander-v2/) 環境。" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "N_4-xJcbBt09" + }, + "source": [ + "%%capture\n", + "import gym\n", + "import random\n", + "import numpy as np\n", + "\n", + "env = gym.make('LunarLander-v2')\n", + "\n", + "fix(env, seed)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "NmiAOfqRwRX5" + }, + "source": [ + "import time\n", + "start = time.time()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "LcMjEUWTBEEB", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "7a5146e4-e877-4d26-fd61-652c57ef1f4e" + }, + "source": [ + "!pip freeze" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "absl-py==0.12.0\n", + "alabaster==0.7.12\n", + "albumentations==0.1.12\n", + "altair==4.1.0\n", + "appdirs==1.4.4\n", + "argon2-cffi==20.1.0\n", + "arviz==0.11.2\n", + "astor==0.8.1\n", + "astropy==4.2.1\n", + "astunparse==1.6.3\n", + "async-generator==1.10\n", + "atari-py==0.2.9\n", + "atomicwrites==1.4.0\n", + "attrs==21.2.0\n", + "audioread==2.1.9\n", + "autograd==1.3\n", + "Babel==2.9.1\n", + "backcall==0.2.0\n", + "beautifulsoup4==4.6.3\n", + "bleach==3.3.0\n", + "blis==0.4.1\n", + "bokeh==2.3.2\n", + "Bottleneck==1.3.2\n", + "box2d-py==2.3.8\n", + "branca==0.4.2\n", + "bs4==0.0.1\n", + "CacheControl==0.12.6\n", + "cached-property==1.5.2\n", + "cachetools==4.2.2\n", + "catalogue==1.0.0\n", + "certifi==2020.12.5\n", + "cffi==1.14.5\n", + "cftime==1.5.0\n", + "chainer==7.4.0\n", + "chardet==3.0.4\n", + "click==7.1.2\n", + "cloudpickle==1.3.0\n", + "cmake==3.12.0\n", + "cmdstanpy==0.9.5\n", + "colorcet==2.0.6\n", + "colorlover==0.3.0\n", + "community==1.0.0b1\n", + "contextlib2==0.5.5\n", + "convertdate==2.3.2\n", + "coverage==3.7.1\n", + "coveralls==0.5\n", + "crcmod==1.7\n", + "cufflinks==0.17.3\n", + "cupy-cuda101==7.4.0\n", + "cvxopt==1.2.6\n", + "cvxpy==1.0.31\n", + "cycler==0.10.0\n", + "cymem==2.0.5\n", + "Cython==0.29.23\n", + "daft==0.0.4\n", + "dask==2.12.0\n", + "datascience==0.10.6\n", + "debugpy==1.0.0\n", + "decorator==4.4.2\n", + "defusedxml==0.7.1\n", + "descartes==1.1.0\n", + "dill==0.3.3\n", + "distributed==1.25.3\n", + "dlib==19.18.0\n", + "dm-tree==0.1.6\n", + "docopt==0.6.2\n", + "docutils==0.17.1\n", + "dopamine-rl==1.0.5\n", + "earthengine-api==0.1.266\n", + "easydict==1.9\n", + "EasyProcess==0.3\n", + "ecos==2.0.7.post1\n", + "editdistance==0.5.3\n", + "en-core-web-sm==2.2.5\n", + "entrypoints==0.3\n", + "ephem==3.7.7.1\n", + "et-xmlfile==1.1.0\n", + "fa2==0.3.5\n", + "fastai==1.0.61\n", + "fastdtw==0.3.4\n", + "fastprogress==1.0.0\n", + "fastrlock==0.6\n", + "fbprophet==0.7.1\n", + "feather-format==0.4.1\n", + "filelock==3.0.12\n", + "firebase-admin==4.4.0\n", + "fix-yahoo-finance==0.0.22\n", + "Flask==1.1.4\n", + "flatbuffers==1.12\n", + "folium==0.8.3\n", + "future==0.16.0\n", + "gast==0.4.0\n", + "GDAL==2.2.2\n", + "gdown==3.6.4\n", + "gensim==3.6.0\n", + "geographiclib==1.50\n", + "geopy==1.17.0\n", + "gin-config==0.4.0\n", + "glob2==0.7\n", + "google==2.0.3\n", + "google-api-core==1.26.3\n", + "google-api-python-client==1.12.8\n", + "google-auth==1.30.0\n", + "google-auth-httplib2==0.0.4\n", + "google-auth-oauthlib==0.4.4\n", + "google-cloud-bigquery==1.21.0\n", + "google-cloud-bigquery-storage==1.1.0\n", + "google-cloud-core==1.0.3\n", + "google-cloud-datastore==1.8.0\n", + "google-cloud-firestore==1.7.0\n", + "google-cloud-language==1.2.0\n", + "google-cloud-storage==1.18.1\n", + "google-cloud-translate==1.5.0\n", + "google-colab==1.0.0\n", + "google-pasta==0.2.0\n", + "google-resumable-media==0.4.1\n", + "googleapis-common-protos==1.53.0\n", + "googledrivedownloader==0.4\n", + "graphviz==0.10.1\n", + "greenlet==1.1.0\n", + "grpcio==1.34.1\n", + "gspread==3.0.1\n", + "gspread-dataframe==3.0.8\n", + "gym==0.17.3\n", + "h5py==3.1.0\n", + "HeapDict==1.0.1\n", + "hijri-converter==2.1.1\n", + "holidays==0.10.5.2\n", + "holoviews==1.14.3\n", + "html5lib==1.0.1\n", + "httpimport==0.5.18\n", + "httplib2==0.17.4\n", + "httplib2shim==0.0.3\n", + "humanize==0.5.1\n", + "hyperopt==0.1.2\n", + "ideep4py==2.0.0.post3\n", + "idna==2.10\n", + "imageio==2.4.1\n", + "imagesize==1.2.0\n", + "imbalanced-learn==0.4.3\n", + "imblearn==0.0\n", + "imgaug==0.2.9\n", + "importlib-metadata==4.0.1\n", + "importlib-resources==5.1.3\n", + "imutils==0.5.4\n", + "inflect==2.1.0\n", + "iniconfig==1.1.1\n", + "install==1.3.4\n", + "intel-openmp==2021.2.0\n", + "intervaltree==2.1.0\n", + "ipykernel==4.10.1\n", + "ipython==5.5.0\n", + "ipython-genutils==0.2.0\n", + "ipython-sql==0.3.9\n", + "ipywidgets==7.6.3\n", + "itsdangerous==1.1.0\n", + "jax==0.2.13\n", + "jaxlib==0.1.66+cuda110\n", + "jdcal==1.4.1\n", + "jedi==0.18.0\n", + "jieba==0.42.1\n", + "Jinja2==2.11.3\n", + "joblib==1.0.1\n", + "jpeg4py==0.1.4\n", + "jsonschema==2.6.0\n", + "jupyter==1.0.0\n", + "jupyter-client==5.3.5\n", + "jupyter-console==5.2.0\n", + "jupyter-core==4.7.1\n", + "jupyterlab-pygments==0.1.2\n", + "jupyterlab-widgets==1.0.0\n", + "kaggle==1.5.12\n", + "kapre==0.3.5\n", + "Keras==2.4.3\n", + "keras-nightly==2.5.0.dev2021032900\n", + "Keras-Preprocessing==1.1.2\n", + "keras-vis==0.4.1\n", + "kiwisolver==1.3.1\n", + "korean-lunar-calendar==0.2.1\n", + "librosa==0.8.0\n", + "lightgbm==2.2.3\n", + "llvmlite==0.34.0\n", + "lmdb==0.99\n", + "LunarCalendar==0.0.9\n", + "lxml==4.2.6\n", + "Markdown==3.3.4\n", + "MarkupSafe==2.0.1\n", + "matplotlib==3.2.2\n", + "matplotlib-inline==0.1.2\n", + "matplotlib-venn==0.11.6\n", + "missingno==0.4.2\n", + "mistune==0.8.4\n", + "mizani==0.6.0\n", + "mkl==2019.0\n", + "mlxtend==0.14.0\n", + "more-itertools==8.7.0\n", + "moviepy==0.2.3.5\n", + "mpmath==1.2.1\n", + "msgpack==1.0.2\n", + "multiprocess==0.70.11.1\n", + "multitasking==0.0.9\n", + "murmurhash==1.0.5\n", + "music21==5.5.0\n", + "natsort==5.5.0\n", + "nbclient==0.5.3\n", + "nbconvert==5.6.1\n", + "nbformat==5.1.3\n", + "nest-asyncio==1.5.1\n", + "netCDF4==1.5.6\n", + "networkx==2.5.1\n", + "nibabel==3.0.2\n", + "nltk==3.2.5\n", + "notebook==5.3.1\n", + "numba==0.51.2\n", + "numexpr==2.7.3\n", + "numpy==1.19.5\n", + "nvidia-ml-py3==7.352.0\n", + "oauth2client==4.1.3\n", + "oauthlib==3.1.0\n", + "okgrade==0.4.3\n", + "opencv-contrib-python==4.1.2.30\n", + "opencv-python==4.1.2.30\n", + "openpyxl==2.5.9\n", + "opt-einsum==3.3.0\n", + "osqp==0.6.2.post0\n", + "packaging==20.9\n", + "palettable==3.3.0\n", + "pandas==1.1.5\n", + "pandas-datareader==0.9.0\n", + "pandas-gbq==0.13.3\n", + "pandas-profiling==1.4.1\n", + "pandocfilters==1.4.3\n", + "panel==0.11.3\n", + "param==1.10.1\n", + "parso==0.8.2\n", + "pathlib==1.0.1\n", + "patsy==0.5.1\n", + "pexpect==4.8.0\n", + "pickleshare==0.7.5\n", + "Pillow==7.1.2\n", + "pip-tools==4.5.1\n", + "plac==1.1.3\n", + "plotly==4.4.1\n", + "plotnine==0.6.0\n", + "pluggy==0.7.1\n", + "pooch==1.3.0\n", + "portpicker==1.3.9\n", + "prefetch-generator==1.0.1\n", + "preshed==3.0.5\n", + "prettytable==2.1.0\n", + "progressbar2==3.38.0\n", + "prometheus-client==0.10.1\n", + "promise==2.3\n", + "prompt-toolkit==1.0.18\n", + "protobuf==3.12.4\n", + "psutil==5.4.8\n", + "psycopg2==2.7.6.1\n", + "ptyprocess==0.7.0\n", + "py==1.10.0\n", + "pyarrow==3.0.0\n", + "pyasn1==0.4.8\n", + "pyasn1-modules==0.2.8\n", + "pycocotools==2.0.2\n", + "pycparser==2.20\n", + "pyct==0.4.8\n", + "pydata-google-auth==1.2.0\n", + "pydot==1.3.0\n", + "pydot-ng==2.0.0\n", + "pydotplus==2.0.2\n", + "PyDrive==1.3.1\n", + "pyemd==0.5.1\n", + "pyerfa==2.0.0\n", + "pyglet==1.5.0\n", + "Pygments==2.6.1\n", + "pygobject==3.26.1\n", + "pymc3==3.11.2\n", + "PyMeeus==0.5.11\n", + "pymongo==3.11.4\n", + "pymystem3==0.2.0\n", + "PyOpenGL==3.1.5\n", + "pyparsing==2.4.7\n", + "pyrsistent==0.17.3\n", + "pysndfile==1.3.8\n", + "PySocks==1.7.1\n", + "pystan==2.19.1.1\n", + "pytest==3.6.4\n", + "python-apt==0.0.0\n", + "python-chess==0.23.11\n", + "python-dateutil==2.8.1\n", + "python-louvain==0.15\n", + "python-slugify==5.0.2\n", + "python-utils==2.5.6\n", + "pytz==2018.9\n", + "PyVirtualDisplay==2.1\n", + "pyviz-comms==2.0.1\n", + "PyWavelets==1.1.1\n", + "PyYAML==3.13\n", + "pyzmq==22.0.3\n", + "qdldl==0.1.5.post0\n", + "qtconsole==5.1.0\n", + "QtPy==1.9.0\n", + "regex==2019.12.20\n", + "requests==2.23.0\n", + "requests-oauthlib==1.3.0\n", + "resampy==0.2.2\n", + "retrying==1.3.3\n", + "rpy2==3.4.4\n", + "rsa==4.7.2\n", + "scikit-image==0.16.2\n", + "scikit-learn==0.22.2.post1\n", + "scipy==1.4.1\n", + "screen-resolution-extra==0.0.0\n", + "scs==2.1.3\n", + "seaborn==0.11.1\n", + "semver==2.13.0\n", + "Send2Trash==1.5.0\n", + "setuptools-git==1.2\n", + "Shapely==1.7.1\n", + "simplegeneric==0.8.1\n", + "six==1.15.0\n", + "sklearn==0.0\n", + "sklearn-pandas==1.8.0\n", + "smart-open==5.0.0\n", + "snowballstemmer==2.1.0\n", + "sortedcontainers==2.4.0\n", + "SoundFile==0.10.3.post1\n", + "spacy==2.2.4\n", + "Sphinx==1.8.5\n", + "sphinxcontrib-serializinghtml==1.1.4\n", + "sphinxcontrib-websupport==1.2.4\n", + "SQLAlchemy==1.4.15\n", + "sqlparse==0.4.1\n", + "srsly==1.0.5\n", + "statsmodels==0.10.2\n", + "sympy==1.7.1\n", + "tables==3.4.4\n", + "tabulate==0.8.9\n", + "tblib==1.7.0\n", + "tensorboard==2.5.0\n", + "tensorboard-data-server==0.6.1\n", + "tensorboard-plugin-wit==1.8.0\n", + "tensorflow==2.5.0\n", + "tensorflow-datasets==4.0.1\n", + "tensorflow-estimator==2.5.0\n", + "tensorflow-gcs-config==2.5.0\n", + "tensorflow-hub==0.12.0\n", + "tensorflow-metadata==1.0.0\n", + "tensorflow-probability==0.12.1\n", + "termcolor==1.1.0\n", + "terminado==0.10.0\n", + "testpath==0.5.0\n", + "text-unidecode==1.3\n", + "textblob==0.15.3\n", + "Theano-PyMC==1.1.2\n", + "thinc==7.4.0\n", + "tifffile==2021.4.8\n", + "toml==0.10.2\n", + "toolz==0.11.1\n", + "torch==1.8.1+cu101\n", + "torchsummary==1.5.1\n", + "torchtext==0.9.1\n", + "torchvision==0.9.1+cu101\n", + "tornado==5.1.1\n", + "tqdm==4.41.1\n", + "traitlets==5.0.5\n", + "tweepy==3.10.0\n", + "typeguard==2.7.1\n", + "typing-extensions==3.7.4.3\n", + "tzlocal==1.5.1\n", + "uritemplate==3.0.1\n", + "urllib3==1.24.3\n", + "vega-datasets==0.9.0\n", + "wasabi==0.8.2\n", + "wcwidth==0.2.5\n", + "webencodings==0.5.1\n", + "Werkzeug==1.0.1\n", + "widgetsnbextension==3.5.1\n", + "wordcloud==1.5.0\n", + "wrapt==1.12.1\n", + "xarray==0.18.2\n", + "xgboost==0.90\n", + "xkit==0.0.0\n", + "xlrd==1.1.0\n", + "xlwt==1.3.0\n", + "yellowbrick==0.9.1\n", + "zict==2.0.0\n", + "zipp==3.4.1\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NrkVvTrvWZ5H" + }, + "source": [ + "## 什麼是 Lunar Lander？\n", + "\n", + "“LunarLander-v2” 這個環境是在模擬登月小艇降落在月球表面時的情形。\n", + "這個任務的目標是讓登月小艇「安全地」降落在兩個黃色旗幟間的平地上。\n", + "> Landing pad is always at coordinates (0,0).\n", + "> Coordinates are the first two numbers in state vector.\n", + "\n", + "![](https://gym.openai.com/assets/docs/aeloop-138c89d44114492fd02822303e6b4b07213010bb14ca5856d2d49d6b62d88e53.svg)\n", + "\n", + "所謂的「環境」其實同時包括了 agent 和 environment。\n", + "我們利用 `step()` 這個函式讓 agent 行動，而後函式便會回傳 environment 給予的 observation/state（以下這兩個名詞代表同樣的意思）和 reward。" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bIbp82sljvAt" + }, + "source": [ + "### Observation / State\n", + "\n", + "首先，我們可以看看 environment 回傳給 agent 的 observation 究竟是長什麼樣子的資料：" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "rsXZra3N9R5T", + "outputId": "9512a449-f90a-4545-8aef-dd9aeb9b2b9e" + }, + "source": [ + "print(env.observation_space)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Box(-inf, inf, (8,), float32)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ezdfoThbAQ49" + }, + "source": [ + "`Box(8,)` 說明我們會拿到 8 維的向量作為 observation，其中包含：垂直及水平座標、速度、角度、加速度等等，這部分我們就不細說。\n", + "\n", + "### Action\n", + "\n", + "而在 agent 得到 observation 和 reward 以後，能夠採取的動作有：" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "p1k4dIrBAaKi", + "outputId": "64cd523a-bbff-4569-cae9-f65123b3c604" + }, + "source": [ + "print(env.action_space)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Discrete(4)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dejXT6PHBrPn" + }, + "source": [ + "`Discrete(4)` 說明 agent 可以採取四種離散的行動：\n", + "- 0 代表不採取任何行動\n", + "- 2 代表主引擎向下噴射\n", + "- 1, 3 則是向左右噴射\n", + "\n", + "接下來，我們嘗試讓 agent 與 environment 互動。\n", + "在進行任何操作前，建議先呼叫 `reset()` 函式讓整個「環境」重置。\n", + "而這個函式同時會回傳「環境」最初始的狀態。" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pi4OmrmZgnWA", + "outputId": "c358ff73-1879-4a74-9579-9ee97740dc16" + }, + "source": [ + "initial_state = env.reset()\n", + "print(initial_state)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "[ 0.00396109 1.4083536 0.40119505 -0.11407257 -0.00458307 -0.09087662\n", + " 0. 0. ]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uBx0mEqqgxJ9" + }, + "source": [ + "接著，我們試著從 agent 的四種行動空間中，隨機採取一個行動" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "vxkOEXRKgizt", + "outputId": "8912cf80-2310-401b-a37e-c0ded59626ee" + }, + "source": [ + "random_action = env.action_space.sample()\n", + "print(random_action)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mns-bO01g0-J" + }, + "source": [ + "再利用 `step()` 函式讓 agent 根據我們隨機抽樣出來的 `random_action` 動作。\n", + "而這個函式會回傳四項資訊：\n", + "- observation / state\n", + "- reward\n", + "- 完成與否\n", + "- 其餘資訊" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "E_WViSxGgIk9" + }, + "source": [ + "observation, reward, done, info = env.step(random_action)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FdieGq7NuBIm" + }, + "source": [ + "第一項資訊 `observation` 即為 agent 採取行動之後，agent 對於環境的 observation 或者說環境的 state 為何。\n", + "而第三項資訊 `done` 則是 `True` 或 `False` 的布林值，當登月小艇成功著陸或是不幸墜毀時，代表這個回合（episode）也就跟著結束了，此時 `step()` 函式便會回傳 `done = True`，而在那之前，`done` 則保持 `False`。" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "yK7r126kuCNp", + "outputId": "3b99114f-e6b4-4a18-c80b-75189083bd55" + }, + "source": [ + "print(done)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "False\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GKdS8vOihxhc" + }, + "source": [ + "### Reward\n", + "\n", + "而「環境」給予的 reward 大致是這樣計算：\n", + "- 小艇墜毀得到 -100 分\n", + "- 小艇在黃旗幟之間成功著地則得 100~140 分\n", + "- 噴射主引擎（向下噴火）每次 -0.3 分\n", + "- 小艇最終完全靜止則再得 100 分\n", + "- 小艇每隻腳碰觸地面 +10 分\n", + "\n", + "> Reward for moving from the top of the screen to landing pad and zero speed is about 100..140 points.\n", + "> If lander moves away from landing pad it loses reward back.\n", + "> Episode finishes if the lander crashes or comes to rest, receiving additional -100 or +100 points.\n", + "> Each leg ground contact is +10.\n", + "> Firing main engine is -0.3 points each frame.\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "vxQNs77hi0_7", + "outputId": "dacd87b3-734e-44f3-c5b4-361b323def84" + }, + "source": [ + "print(reward) # after doing a random action (0), the immediate reward is stored in this " + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "-0.8588900517154912\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Mhqp6D-XgHpe" + }, + "source": [ + "### Random Agent\n", + "\n", + "最後，在進入實做之前，我們就來看看這樣一個 random agent 能否成功登陸月球：" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 269 + }, + "id": "Y3G0bxoccelv", + "outputId": "36096915-445e-40fb-b349-a6a9a5b900d5" + }, + "source": [ + "\n", + "env.reset()\n", + "\n", + "img = plt.imshow(env.render(mode='rgb_array'))\n", + "\n", + "done = False\n", + "while not done:\n", + " action = env.action_space.sample()\n", + " observation, reward, done, _ = env.step(action)\n", + "\n", + " img.set_data(env.render(mode='rgb_array'))\n", + " display.display(plt.gcf())\n", + " display.clear_output(wait=True)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAd2ElEQVR4nO3de3RW9b3n8fc39xCu4RLC3SCtRYvcL8taKD0cKWtmsC0qTr3AaCnWarvmTOfomTVHzzmrx1W0doZlh5baKrQVag9eWBZQBI8VKSAochWIIVxyAgECgYDkxnf+eHbCIwnk9iRPdvJ5rfWs7P3bez/7+wvP82Hn9+z9bHN3REQkPBLiXYCIiDSOgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREKmxYLbzKab2T4zyzWzx1pqPyIiHY21xHncZpYI7AemAUeBD4C73X1PzHcmItLBtNQR93gg193z3L0cWA7MbKF9iYh0KEkt9Lz9gSNR80eBCVdb2cx0+aaIyBXc3epqb6ngrpeZzQPmxWv/IiJh1VLBXQAMjJofELTVcPfFwGLQEbeISGO01Bj3B8AwM7vOzFKA2cDKFtqXiEiH0iJH3O5eaWY/AN4EEoHfuvvultiXiEhH0yKnAza6CA2ViIjUcrUPJ3XlpIhIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREKmWfecNLN84BxQBVS6+1gzywT+CAwB8oE73f1088oUEZFqsTji/pq7j3T3scH8Y8A6dx8GrAvmRUQkRlpiqGQmsCSYXgLc3gL7EBHpsJob3A68ZWbbzGxe0Jbl7oXB9DEgq5n7EBGRKM0a4wa+4u4FZtYHWGtmn0QvdHc3M69rwyDo59W1TERErs7c68zVxj+R2ZNAKfBdYIq7F5pZNvDv7v7FeraNTREiIu2Iu1td7U0eKjGzDDPrUj0N/C2wC1gJ3B+sdj/welP3ISIitTX5iNvMcoBXg9kk4CV3/4mZ9QReBgYBh4icDlhcz3PpiFtE5ApXO+KO2VBJcyi4RURqi/lQiYiIxIeCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiFTb3Cb2W/NrMjMdkW1ZZrZWjM7EPzsEbSbmS00s1wz22Fmo1uyeBGRjqghR9wvAtOvaHsMWOfuw4B1wTzAN4BhwWMesCg2ZYqISLV6g9vd/wIUX9E8E1gSTC8Bbo9qX+oRm4DuZpYdq2JFRKTpY9xZ7l4YTB8DsoLp/sCRqPWOBm21mNk8M9tqZlubWIOISIeU1NwncHc3M2/CdouBxQBN2V5EpKNq6hH38eohkOBnUdBeAAyMWm9A0CYiIjHS1OBeCdwfTN8PvB7Vfl9wdslEoCRqSEVERGLA3K89SmFmy4ApQC/gOPAE8BrwMjAIOATc6e7FZmbAc0TOQrkAzHX3esewNVQiIlKbu1td7fUGd2tQcIuI1Ha14NaVkyIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQmZeoPbzH5rZkVmtiuq7UkzKzCz7cFjRtSyx80s18z2mdltLVW4iEhH1ZCbBX8VKAWWuvtNQduTQKm7P3PFusOBZcB4oB/wNvAFd6+qZx+656SIyBWafM9Jd/8LUNzA/cwElrt7mbsfBHKJhLiIiMRIc8a4f2BmO4KhlB5BW3/gSNQ6R4O2WsxsnpltNbOtzahBRKTDaWpwLwKGAiOBQuBnjX0Cd1/s7mPdfWwTaxAR6ZCaFNzuftzdq9z9EvBrLg+HFAADo1YdELSJiEiMNCm4zSw7avabQPUZJyuB2WaWambXAcOALc0rUUREoiXVt4KZLQOmAL3M7CjwBDDFzEYCDuQD3wNw991m9jKwB6gEHq7vjBIREWmcek8HbJUidDqgiEgtTT4dUERE2hYFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjI1BvcZjbQzN4xsz1mttvMfhi0Z5rZWjM7EPzsEbSbmS00s1wz22Fmo1u6EyIiHUlDjrgrgb9z9+HAROBhMxsOPAasc/dhwLpgHuAbRO7uPgyYByyKedUiIh1YvcHt7oXu/mEwfQ7YC/QHZgJLgtWWALcH0zOBpR6xCehuZtkxr1xEpINq1Bi3mQ0BRgGbgSx3LwwWHQOygun+wJGozY4GbVc+1zwz22pmWxtZs4hIh9bg4DazzsAK4EfufjZ6mbs74I3Zsbsvdvex7j62MduJiHR0DQpuM0smEtp/cPdXgubj1UMgwc+ioL0AGBi1+YCgTUREYqAhZ5UY8Btgr7s/G7VoJXB/MH0/8HpU+33B2SUTgZKoIRUREWkmi4xyXGMFs68A7wE7gUtB8z8QGed+GRgEHALudPfiIOifA6YDF4C57n7NcWwza9Qwi4hIR+DuVld7vcHdGhTcIiK1XS24deWkiEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQqYhNwseaGbvmNkeM9ttZj8M2p80swIz2x48ZkRt87iZ5ZrZPjO7rSU7ICLS0TTkZsHZQLa7f2hmXYBtwO3AnUCpuz9zxfrDgWXAeKAf8DbwBXevusY+dM9JEZErNPmek+5e6O4fBtPngL1A/2tsMhNY7u5l7n4QyCUS4iIiEgONGuM2syHAKGBz0PQDM9thZr81sx5BW3/gSNRmR7l20IsA8K//+j1++lO46SYYPhz69Yt3Ra1vypQpvPjiF5kxA268EW64ARIT412VtDVJDV3RzDoDK4AfuftZM1sE/Avgwc+fAf+tEc83D5jXuHKlPfvyl3PIzoapUyPzhYWwZ09kes0ayM0Fdzh2DKquOvAWbr1792b8+FJuvDEyX1kJGzdCRQUcPQqvvRZpLymBc+fiV6fEV4OC28ySiYT2H9z9FQB3Px61/NfAG8FsATAwavMBQdvnuPtiYHGwvca4pYYFo3r9+l0+6v7a1yKhXVUFb74Jn30WCfbf/z5+dbak6t9BcjJMnhyZdod77olM79oF+/ZFppcuhePHaz+HtF8NOavEgN8Ae9392aj27KjVvgnsCqZXArPNLNXMrgOGAVtiV7J0RJcuRUK7shIuXIDz5yPh3ZFU/8dVVQUXL0Z+B+fPR3430rE05Ij7FuBeYKeZbQ/a/gG428xGEhkqyQe+B+Duu83sZWAPUAk8fK0zSkSiuUceEBka2B684t58E/LyIsuKi9t/WFX/HiorYf16KC+HggJYuTKyvLS04/3HJZfVG9zuvgGo65SUVdfY5ifAT5pRl3RApaXw5z9Hhj8uXYqM4Z44Ee+qWt/27fDrX8OhQ5Hfw+HD7f8/KmmcBn84KdLSDh+GJ5+MdxXx9+yzsHVrvKuQtkyXvIuIhIyCW0QkZDRUIhITRkJC5EoZ9yrq+yoJkeZQcIvEwLBhX2Xw4LEAHDu2l5KSQgAuXDjDqVMH41matEMKbpFmMjP69x/Bl/rOpFNyJqd7H+R8eeR0mM/KT3Pq7KcAVFSUkZf3PpeCU0SKiw9TWXkxbnVLeCm4RZopK+uL9On2RbqnDSIxIYX05MyaZRVVF7mYeRqAKi9nUN9xuEeC+/ipPZRVnAfg8OGtFBTsbP3iJZQU3CLN1KVLHzql9iYxIaXWsuTENJITL19k3C11UM10/67jqbpURtH5XRQVHWiVWqV9UHCLNENiYgpDhkwgK+PLDVrf7PK1bF1TI1/EcuZifkuUJu2YTgcUaYYePQbSNb0fyYlp8S5FOhAFt8hVdE1KYnT37jXf99A5LY2BPXuSknT5D9WePQfTNa0/SQkKbmk9GioRqUPXpCS+m5PDdRkZ/LmwkLVFRYzOyaFrejonz53jw7w8SEhh0MAx9MkYHu9ypYPREbdIHbLS0hjUqRPJCQnc3L07iWY1p/FVVUUusBk8eCz9e4wmNalbTPfdtWsWSUk6gperU3CL1OFAaSk7S0o4cO4cv87Lo8KNqi4j8B5jOXz2EhVVVaSndyclqQsJFrt7i6WkZDBu9HcYNfLbJCWlxux5pX3RUEkb0L17d1JTUyktLeX8+fPxLkcCvzkYueLRgb59v8SAfrfSK+MLVI3sxYa/LiYxMZkEi91byCyBnJyJDEj7GwqGVFB0Yh+HDulrAqU2BXecJCUlceONNzJz5ky+9a1vMXToUDZs2MD777/PCy+8QGFhYc2f5hIf1d82kpycRs6QSWR3HYmRQEnpfwAwqP8Yene6IXb7c+fUqYN061XAoG6TGHljIefOnaC4+FDM9iHtg4K7lWVkZHDHHXcwceJE7rjjDjIzL19lN336dG677TYeeeQRFi1axM6dO3n11VcV4HE2bNhkhg/+z6Ql9WBXwcvs3r0aALNEzGI52uicOJFH1077MR9Fv+6jGTJkPGfOFHDpUmUM9yNhp+BuJZmZmXzzm9/kxz/+McOGDSMhoe43vJnRp08fnnjiCcrKyti+fTsLFizgwIED7NypS6JbW58+X+BLOdPp0/lGjpXu4NNDGzh79hidO/dusX3u2b+ajKSzDOs5nS8P/RZnzx4nN/cvLbY/CZ96g9vM0oC/AKnB+v/m7k8ENwJeDvQEtgH3unu5maUCS4ExwCngLnfPb6H627SEhATGjBnD448/ztChQxkxYkSjtk9NTWXChAmsWLGCwsJCXnvtNX73u9/xySefcPr06RaqWqolJaUwbNhkBmZO4JJXUnD6Qw4d2gZAnz7XU+kXOXBqNX07jyIxIfJWSkvqQVJC8z5UPHEil/8ov0hm+nX07TyCG4b+DSdP5nHmzNFm90nah4YccZcBU9291MySgQ1mthr478DP3X25mf0SeABYFPw87e7Xm9ls4KfAXS1Uf5vUv39/7rvvPiZNmsS0adNIS2v+qV3Z2dnMnz+f+fPns2nTJn7xi1+wZcsW8vLyqKrSvZhbQq9eOQzMGkPnlL7sKXqND7b+gYsXSwA4dGgb588Xk5CQxNCht9AlNZ1eaal4ykBI7AxARnIW3dOGAJBgCaQkdvncJe8AZZXnOFX6KceP76tpq6i4yN5P1tK3z5cY0e87DOn9VU584QAfbF2mIRMBGnazYAdKg9nk4OHAVOC/Bu1LgCeJBPfMYBrg34DnzMy8nX+zfHp6OuPHj+fuu+9m2rRpDBky5KrDIU1V/aafNGkSEyZMoLy8nGXLlvH0009z7NgxHYXHUEZGL0aNmMWArhM4eX4fBw9v4MyZgprlVVXlNWFbWLiHvunpjOmZSb5155xFjri7d+9PVtYXAUhOTKdP18iFOobRJ+MmkhJSKas6x2flZ7hwofhz+z9//hRbP1pORlof+mR8idTULiQmpii4BWjgGLeZJRIZDrke+AXwKXDG3atfRUeB/sF0f+AIgLtXmlkJkeGUkzGsu00wM3r16sV3v/tdxowZw4wZM2JydN0QCQkJpKWlMWfOHO666y4OHTrEc889x5IlSygvL6eioqJV6oiVtLQ0Nm7cSEZGRhu5e8xFjp3YTueuXbhYdpa8g/9OWloKUPsbAAHOAu8UFwOXA7i4eD/5+ZGx6ZSUTgwYMBKIvG4GDxpHcnI6AGlpyWRkZFBVVUl+fj4lJSV06tSJM2cOs333y/ToMZBDh7dSUXGhJTssIWKNeZOYWXfgVeB/Ay+6+/VB+0BgtbvfZGa7gOnufjRY9ikwwd1PXvFc84B5weyYZvekFSUkJJCVlcWcOXN49NFH6dOnT8yPrpuioqKCU6dOsXbtWn71q1/x17/+tc2dkVL9e+ratSuzZs2qmf/+979PVlZWPEurxSyRlJR03J3y8tieX5+S0qnmjBT3S5SX1w7lV155hY8//pjExBSqqsp57bXXOHny8tuorf3bSuy5u9XV3qjgBjCzfwQ+A/4e6BscVU8CnnT328zszWD6r2aWBBwDel9rqCQ9Pd0vXmz7dwJJTExk2rRpPPbYYwwZMoTBgwfHu6SrKi4upqioiOeee44//elPFBUVxaWOTp068fWvf52EhATMjEcffZTs7GySk5PJycmpNeYrdXN3Dh06RPX75OLFiyxYsIALFy4H/rZt2zh6VB9gtidNDm4z6w1UuPsZM0sH3iLygeP9wIqoDyd3uPv/M7OHgS+7+/zgw8lvufud19rHzTff7IsWLaqZLy4u5umnn6750O3MmTPs3r27wZ2NtRtuuIEHH3yQW265hdGjR5OSUvefy23Vnj17yM/P56mnnmLTpk1UVsZ+nDQ1NZUxY8ZgZowePZrZs2fXtI8aNapN/EXS3u3fv7/miNzdWbhwIQUFl8flCwoKyM/Pj1N10hTNCe4RRD58TCTy3SYvu/s/m1kOkdMBM4GPgHvcvSw4ffB3wCgiA36z3T3vWvsYO3asb9169Ut7i4qK2LRpU818ZWUlzzzzDGfOnKlpi/WHc926dWPq1KnMmTOHcePGkZ2dXf9GbVxFRQVvv/02W7du5fnnn+fw4cONfo5+/frRrVvkS5VGjhxZE9CdOnVi6tSpCug2LDc3lz179tTML1u2jI8//rhmPi8vj7KysniUJlcRs6GSllBfcF/J3WuN723atOlzL8oXXniBgwcv3137s88+o6Sk5JrPm5iYSO/evZk8eTIPPfQQt956a7sMInenoKCA1atX88orr/Dee+/V+o6Ubt26kZ4e+fBsxIgRzJo1C4ApU6aQk5MDRD5ka4+/n47i0qVLn/sg+PXXX+fUqVMArFmzho0bNzbofSMtp10Fd0NcvHjxc+c37969m9WrV9fMf/bZZyxevJjy8nIgcp70I488wty5c0lNTQ3dcEhTlZWVsX79erZs2fK59hkzZjB8eOT0taSkJFJT9U11HUn1mUl79uxh1apV/PGPf+Tw4cOUlZW1yFCb1K3DBXd9Ll26xMmTJ2uO3JOTk8nMzNSHZSJ1KC4upry8nNWrV7N582YOHDjAu+++q4u/WpiCW0RipqSkhGPHjrF48WLy8vLYsWMHeXnX/ChLmkDBLSIt5uDBgxw5coQFCxZQUlLCjh07OHv2bLzLCj0Ft4i0mo0bN7Jv3z6effZZKisryc3N1dh4Eyi4RaRVVZ/9VVVVxYoVK/joo4/4/e9/T2VlJSdOnIh3eaGg4BaRuKqqqqKsrIzTp0/z4osv8v777/Pee+9RWVlJGK6cjgcFt4i0KefPn6e0tJT9+/fz0ksvsXbtWvLz82udX96RXS24dQccEYmLjIwMMjIyyMrK4tZbb6WgoIDS0lLWr1/PW2+9xbp16zh37ly8y2yTdMQtIm2Ou7Njxw5+9rOfsWrVqporOjuaqx1x63plEWlzzIybb76ZpUuXsnr1aubOnUvv3i13n8+wUXCLSJs2btw4nn/+edasWcNDDz1EZmZmvEuKOwW3iLR5CQkJjB49moULF/Luu+8yf/58OnXqFO+y4kbBLSKhkZSUxE033cTChQv54IMPmDt3bqvdLrAtUXCLSOgkJyczfPhwFi9ezLZt27j33nvp3LlzvMtqNQpuEQmtpKQkhg8fzpIlS9iwYQP33ntvhxgDV3CLSOhFn4WyZs0aXnrpJUaMGEFiYmK8S2sRCm4RaVfGjRvH7Nmz2bJlC0uXLmXEiBHt7sYo9Qa3maWZ2RYz+9jMdpvZPwXtL5rZQTPbHjxGBu1mZgvNLNfMdpjZ6JbuhIhINDMjNTWVu+++m82bN/PLX/6SUaNGkZycHO/SYqIhR9xlwFR3vxkYCUw3s4nBsh+7+8jgsT1o+wYwLHjMAxbVekYRkVZgZqSlpTF37lzeeecdFi1axPjx40N/r9R6q/eI0mA2OXhc6zr5mcDSYLtNQHczC/8t0kUk1Lp168YDDzzAqlWrWLJkCZMmTYp3SU3WoP92zCzRzLYDRcBad98cLPpJMBzyczOrvptsf+BI1OZHgzYRkbjr2bMn99xzD2+88QbLly9n3Lhx8S6pRo8ePZg8eTKTJ0++5umNDfp2QHevAkaaWXfgVTO7CXgcOAakAIuBvwf+uaEFmtk8IkMpDBo0qKGbiYjERGZmJnfddRczZsxg1apVPPXUU+zatSumN0DOyMggJyenVvv111/Pgw8+WKu9V69ejB8/HoCxY8de9Xkb9bWu7n7GzN4Bprv7M0FzmZm9APyPYL4AGBi12YCg7crnWkwk8Bk7dmz8v6JQRDqkLl26cOedd3L77bezYsUKFixYwN69eykvL6+1bmpqap1fdnXdddfxwAMP1Grv27cv06ZNq9VuZpjV+cV/DVJvcJtZb6AiCO10YBrwUzPLdvdCi+z9dmBXsMlK4AdmthyYAJS4e2GTKxQRaWHRZ6F8+9vfZtmyZRw7dqzWegMGDGDWrFm12hMSElr1lMOGHHFnA0vMLJHImPjL7v6Gma0PQt2A7cD8YP1VwAwgF7gAzI192SIisVcd4HPmzIl3KddUb3C7+w5gVB3tU6+yvgMPN780ERGpS7hPZhQR6YAU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMubu8a4BMzsH7It3HS2kF3Ay3kW0gPbaL2i/fVO/wmWwu/eua0FSa1dyFfvcfWy8i2gJZra1PfatvfYL2m/f1K/2Q0MlIiIho+AWEQmZthLci+NdQAtqr31rr/2C9ts39audaBMfToqISMO1lSNuERFpoLgHt5lNN7N9ZpZrZo/Fu57GMrPfmlmRme2Kass0s7VmdiD42SNoNzNbGPR1h5mNjl/l12ZmA83sHTPbY2a7zeyHQXuo+2ZmaWa2xcw+Dvr1T0H7dWa2Oaj/j2aWErSnBvO5wfIh8ay/PmaWaGYfmdkbwXx76Ve+me00s+1mtjVoC/VrsTniGtxmlgj8AvgGMBy428yGx7OmJngRmH5F22PAOncfBqwL5iHSz2HBYx6wqJVqbIpK4O/cfTgwEXg4+LcJe9/KgKnufjMwEphuZhOBnwI/d/frgdPAA8H6DwCng/afB+u1ZT8E9kbNt5d+AXzN3UdGnfoX9tdi07l73B7AJODNqPnHgcfjWVMT+zEE2BU1vw/IDqaziZynDvAr4O661mvrD+B1YFp76hvQCfgQmEDkAo6koL3mdQm8CUwKppOC9SzetV+lPwOIBNhU4A3A2kO/ghrzgV5XtLWb12JjH/EeKukPHImaPxq0hV2WuxcG08eArGA6lP0N/oweBWymHfQtGE7YDhQBa4FPgTPuXhmsEl17Tb+C5SVAz9atuMH+D/A/gUvBfE/aR78AHHjLzLaZ2bygLfSvxaZqK1dOtlvu7mYW2lN3zKwzsAL4kbufNbOaZWHtm7tXASPNrDvwKnBDnEtqNjP7T0CRu28zsynxrqcFfMXdC8ysD7DWzD6JXhjW12JTxfuIuwAYGDU/IGgLu+Nmlg0Q/CwK2kPVXzNLJhLaf3D3V4LmdtE3AHc/A7xDZAihu5lVH8hE117Tr2B5N+BUK5faELcA/8XM8oHlRIZL/i/h7xcA7l4Q/Cwi8p/teNrRa7Gx4h3cHwDDgk++U4DZwMo41xQLK4H7g+n7iYwPV7ffF3zqPREoifpTr02xyKH1b4C97v5s1KJQ983MegdH2phZOpFx+71EAnxWsNqV/aru7yxgvQcDp22Juz/u7gPcfQiR99F6d/8OIe8XgJllmFmX6mngb4FdhPy12CzxHmQHZgD7iYwz/q9419OE+pcBhUAFkbG0B4iMFa4DDgBvA5nBukbkLJpPgZ3A2HjXf41+fYXIuOIOYHvwmBH2vgEjgI+Cfu0C/jFozwG2ALnAn4DUoD0tmM8NlufEuw8N6OMU4I320q+gDx8Hj93VORH212JzHrpyUkQkZOI9VCIiIo2k4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZP4/aOAjA9p4Wi8AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "F5paWqo7tWL2" + }, + "source": [ + "## Policy Gradient\n", + "\n", + "現在來搭建一個簡單的 policy network。\n", + "我們預設模型的輸入是 8-dim 的 observation，輸出則是離散的四個動作之一：" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "J8tdmeD-tZew" + }, + "source": [ + "class PolicyGradientNetwork(nn.Module):\n", + "\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.fc1 = nn.Linear(8, 16)\n", + " self.fc2 = nn.Linear(16, 16)\n", + " self.fc3 = nn.Linear(16, 4)\n", + "\n", + " def forward(self, state):\n", + " hid = torch.tanh(self.fc1(state))\n", + " hid = torch.tanh(self.fc2(hid))\n", + " return F.softmax(self.fc3(hid), dim=-1)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ynbqJrhIFTC3" + }, + "source": [ + "再來，搭建一個簡單的 agent，並搭配上方的 policy network 來採取行動。\n", + "這個 agent 能做到以下幾件事：\n", + "- `learn()`：從記下來的 log probabilities 及 rewards 來更新 policy network。\n", + "- `sample()`：從 environment 得到 observation 之後，利用 policy network 得出應該採取的行動。\n", + "而此函式除了回傳抽樣出來的 action，也會回傳此次抽樣的 log probabilities。" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "zZo-IxJx286z" + }, + "source": [ + "\n", + "class PolicyGradientAgent():\n", + " \n", + " def __init__(self, network):\n", + " self.network = network\n", + " self.optimizer = optim.SGD(self.network.parameters(), lr=0.001)\n", + " \n", + " def forward(self, state):\n", + " return self.network(state)\n", + " def learn(self, log_probs, rewards):\n", + " loss = (-log_probs * rewards).sum() # You don't need to revise this to pass simple baseline (but you can)\n", + "\n", + " self.optimizer.zero_grad()\n", + " loss.backward()\n", + " self.optimizer.step()\n", + " \n", + " def sample(self, state):\n", + " action_prob = self.network(torch.FloatTensor(state))\n", + " action_dist = Categorical(action_prob)\n", + " action = action_dist.sample()\n", + " log_prob = action_dist.log_prob(action)\n", + " return action.item(), log_prob\n", + "\n", + " def save(self, PATH): # You should not revise this\n", + " Agent_Dict = {\n", + " \"network\" : self.network.state_dict(),\n", + " \"optimizer\" : self.optimizer.state_dict()\n", + " }\n", + " torch.save(Agent_Dict, PATH)\n", + "\n", + " def load(self, PATH): # You should not revise this\n", + " checkpoint = torch.load(PATH)\n", + " self.network.load_state_dict(checkpoint[\"network\"])\n", + " #如果要儲存過程或是中斷訓練後想繼續可以用喔 ^_^\n", + " self.optimizer.load_state_dict(checkpoint[\"optimizer\"])\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ehPlnTKyRZf9" + }, + "source": [ + "最後，建立一個 network 和 agent，就可以開始進行訓練了。" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "GfJIvML-RYjL" + }, + "source": [ + "network = PolicyGradientNetwork()\n", + "agent = PolicyGradientAgent(network)\n", + "#agent = PolicyGradientAgent()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ouv23glgf5Qt" + }, + "source": [ + "## 訓練 Agent\n", + "\n", + "現在我們開始訓練 agent。\n", + "透過讓 agent 和 environment 互動，我們記住每一組對應的 log probabilities 及 reward，並在成功登陸或者不幸墜毀後，回放這些「記憶」來訓練 policy network。" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000, + "referenced_widgets": [ + "2acab9542fe64b979fa2ac2adb3f10a8", + "f288c64b5ff748eb82178bf1de17934f", + "de34e5b178f5470e98e0275102a65042", + "c93cba301cac439ca56fb6b45bd1c4e4", + "43c6ee720b674626ab3a869bda5dd6e3", + "2465d2b109d34922a486341232d86ad6", + "aa27187195be4da9874025395eac35eb", + "02d196d4f9734f998455d92bd9300adb" + ] + }, + "id": "vg5rxBBaf38_", + "outputId": "eae0c9f4-0efc-40fe-a29e-7f7194613f6d" + }, + "source": [ + "agent.network.train() # 訓練前，先確保 network 處在 training 模式\n", + "EPISODE_PER_BATCH = 5 # 每蒐集 5 個 episodes 更新一次 agent\n", + "NUM_BATCH = 400 # 總共更新 400 次\n", + "\n", + "avg_total_rewards, avg_final_rewards = [], []\n", + "\n", + "prg_bar = tqdm(range(NUM_BATCH))\n", + "for batch in prg_bar:\n", + "\n", + " log_probs, rewards = [], []\n", + " total_rewards, final_rewards = [], []\n", + "\n", + " # 蒐集訓練資料\n", + " for episode in range(EPISODE_PER_BATCH):\n", + " \n", + " state = env.reset()\n", + " total_reward, total_step = 0, 0\n", + " seq_rewards = []\n", + " while True:\n", + "\n", + " action, log_prob = agent.sample(state) # at , log(at|st)\n", + " next_state, reward, done, _ = env.step(action)\n", + "\n", + " log_probs.append(log_prob) # [log(a1|s1), log(a2|s2), ...., log(at|st)]\n", + " # seq_rewards.append(reward)\n", + " state = next_state\n", + " total_reward += reward\n", + " total_step += 1\n", + " rewards.append(reward) #改這裡\n", + " # ! 重要 ！\n", + " # 現在的reward 的implementation 為每個時刻的瞬時reward, 給定action_list : a1, a2, a3 ......\n", + " # reward : r1, r2 ,r3 ......\n", + " # medium：將reward調整成accumulative decaying reward, 給定action_list : a1, a2, a3 ......\n", + " # reward : r1+0.99*r2+0.99^2*r3+......, r2+0.99*r3+0.99^2*r4+...... ,r3+0.99*r4+0.99^2*r5+ ......\n", + " # boss : implement DQN\n", + " if done:\n", + " final_rewards.append(reward)\n", + " total_rewards.append(total_reward)\n", + " break\n", + "\n", + " print(f\"rewards looks like \", np.shape(rewards)) \n", + " print(f\"log_probs looks like \", np.shape(log_probs)) \n", + " # 紀錄訓練過程\n", + " avg_total_reward = sum(total_rewards) / len(total_rewards)\n", + " avg_final_reward = sum(final_rewards) / len(final_rewards)\n", + " avg_total_rewards.append(avg_total_reward)\n", + " avg_final_rewards.append(avg_final_reward)\n", + " prg_bar.set_description(f\"Total: {avg_total_reward: 4.1f}, Final: {avg_final_reward: 4.1f}\")\n", + "\n", + " # 更新網路\n", + " # rewards = np.concatenate(rewards, axis=0)\n", + " rewards = (rewards - np.mean(rewards)) / (np.std(rewards) + 1e-9) # 將 reward 正規標準化\n", + " agent.learn(torch.stack(log_probs), torch.from_numpy(rewards))\n", + " print(\"logs prob looks like \", torch.stack(log_probs).size())\n", + " print(\"torch.from_numpy(rewards) looks like \", torch.from_numpy(rewards).size())" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2acab9542fe64b979fa2ac2adb3f10a8", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=400.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "rewards looks like (448,)\n", + "log_probs looks like (448,)\n", + "logs prob looks like torch.Size([448])\n", + "torch.from_numpy(rewards) looks like torch.Size([448])\n", + "rewards looks like (515,)\n", + "log_probs looks like (515,)\n", + "logs prob looks like torch.Size([515])\n", + "torch.from_numpy(rewards) looks like torch.Size([515])\n", + "rewards looks like (392,)\n", + "log_probs looks like (392,)\n", + "logs prob looks like torch.Size([392])\n", + "torch.from_numpy(rewards) looks like torch.Size([392])\n", + "rewards looks like (518,)\n", + "log_probs looks like (518,)\n", + "logs prob looks like torch.Size([518])\n", + "torch.from_numpy(rewards) looks like torch.Size([518])\n", + "rewards looks like (472,)\n", + "log_probs looks like (472,)\n", + "logs prob looks like torch.Size([472])\n", + "torch.from_numpy(rewards) looks like torch.Size([472])\n", + "rewards looks like (530,)\n", + "log_probs looks like (530,)\n", + "logs prob looks like torch.Size([530])\n", + "torch.from_numpy(rewards) looks like torch.Size([530])\n", + "rewards looks like (463,)\n", + "log_probs looks like (463,)\n", + "logs prob looks like torch.Size([463])\n", + "torch.from_numpy(rewards) looks like torch.Size([463])\n", + "rewards looks like (540,)\n", + "log_probs looks like (540,)\n", + "logs prob looks like torch.Size([540])\n", + "torch.from_numpy(rewards) looks like torch.Size([540])\n", + "rewards looks like (513,)\n", + "log_probs looks like (513,)\n", + "logs prob looks like torch.Size([513])\n", + "torch.from_numpy(rewards) looks like torch.Size([513])\n", + "rewards looks like (449,)\n", + "log_probs looks like (449,)\n", + "logs prob looks like torch.Size([449])\n", + "torch.from_numpy(rewards) looks like torch.Size([449])\n", + "rewards looks like (602,)\n", + "log_probs looks like (602,)\n", + "logs prob looks like torch.Size([602])\n", + "torch.from_numpy(rewards) looks like torch.Size([602])\n", + "rewards looks like (542,)\n", + "log_probs looks like (542,)\n", + "logs prob looks like torch.Size([542])\n", + "torch.from_numpy(rewards) looks like torch.Size([542])\n", + "rewards looks like (503,)\n", + "log_probs looks like (503,)\n", + "logs prob looks like torch.Size([503])\n", + "torch.from_numpy(rewards) looks like torch.Size([503])\n", + "rewards looks like (470,)\n", + "log_probs looks like (470,)\n", + "logs prob looks like torch.Size([470])\n", + "torch.from_numpy(rewards) looks like torch.Size([470])\n", + "rewards looks like (518,)\n", + "log_probs looks like (518,)\n", + "logs prob looks like torch.Size([518])\n", + "torch.from_numpy(rewards) looks like torch.Size([518])\n", + "rewards looks like (421,)\n", + "log_probs looks like (421,)\n", + "logs prob looks like torch.Size([421])\n", + "torch.from_numpy(rewards) looks like torch.Size([421])\n", + "rewards looks like (592,)\n", + "log_probs looks like (592,)\n", + "logs prob looks like torch.Size([592])\n", + "torch.from_numpy(rewards) looks like torch.Size([592])\n", + "rewards looks like (520,)\n", + "log_probs looks like (520,)\n", + "logs prob looks like torch.Size([520])\n", + "torch.from_numpy(rewards) looks like torch.Size([520])\n", + "rewards looks like (494,)\n", + "log_probs looks like (494,)\n", + "logs prob looks like torch.Size([494])\n", + "torch.from_numpy(rewards) looks like torch.Size([494])\n", + "rewards looks like (461,)\n", + "log_probs looks like (461,)\n", + "logs prob looks like torch.Size([461])\n", + "torch.from_numpy(rewards) looks like torch.Size([461])\n", + "rewards looks like (572,)\n", + "log_probs looks like (572,)\n", + "logs prob looks like torch.Size([572])\n", + "torch.from_numpy(rewards) looks like torch.Size([572])\n", + "rewards looks like (593,)\n", + "log_probs looks like (593,)\n", + "logs prob looks like torch.Size([593])\n", + "torch.from_numpy(rewards) looks like torch.Size([593])\n", + "rewards looks like (569,)\n", + "log_probs looks like (569,)\n", + "logs prob looks like torch.Size([569])\n", + "torch.from_numpy(rewards) looks like torch.Size([569])\n", + "rewards looks like (546,)\n", + "log_probs looks like (546,)\n", + "logs prob looks like torch.Size([546])\n", + "torch.from_numpy(rewards) looks like torch.Size([546])\n", + "rewards looks like (612,)\n", + "log_probs looks like (612,)\n", + "logs prob looks like torch.Size([612])\n", + "torch.from_numpy(rewards) looks like torch.Size([612])\n", + "rewards looks like (534,)\n", + "log_probs looks like (534,)\n", + "logs prob looks like torch.Size([534])\n", + "torch.from_numpy(rewards) looks like torch.Size([534])\n", + "rewards looks like (513,)\n", + "log_probs looks like (513,)\n", + "logs prob looks like torch.Size([513])\n", + "torch.from_numpy(rewards) looks like torch.Size([513])\n", + "rewards looks like (513,)\n", + "log_probs looks like (513,)\n", + "logs prob looks like torch.Size([513])\n", + "torch.from_numpy(rewards) looks like torch.Size([513])\n", + "rewards looks like (535,)\n", + "log_probs looks like (535,)\n", + "logs prob looks like torch.Size([535])\n", + "torch.from_numpy(rewards) looks like torch.Size([535])\n", + "rewards looks like (533,)\n", + "log_probs looks like (533,)\n", + "logs prob looks like torch.Size([533])\n", + "torch.from_numpy(rewards) looks like torch.Size([533])\n", + "rewards looks like (521,)\n", + "log_probs looks like (521,)\n", + "logs prob looks like torch.Size([521])\n", + "torch.from_numpy(rewards) looks like torch.Size([521])\n", + "rewards looks like (566,)\n", + "log_probs looks like (566,)\n", + "logs prob looks like torch.Size([566])\n", + "torch.from_numpy(rewards) looks like torch.Size([566])\n", + "rewards looks like (586,)\n", + "log_probs looks like (586,)\n", + "logs prob looks like torch.Size([586])\n", + "torch.from_numpy(rewards) looks like torch.Size([586])\n", + "rewards looks like (575,)\n", + "log_probs looks like (575,)\n", + "logs prob looks like torch.Size([575])\n", + "torch.from_numpy(rewards) looks like torch.Size([575])\n", + "rewards looks like (709,)\n", + "log_probs looks like (709,)\n", + "logs prob looks like torch.Size([709])\n", + "torch.from_numpy(rewards) looks like torch.Size([709])\n", + "rewards looks like (486,)\n", + "log_probs looks like (486,)\n", + "logs prob looks like torch.Size([486])\n", + "torch.from_numpy(rewards) looks like torch.Size([486])\n", + "rewards looks like (557,)\n", + "log_probs looks like (557,)\n", + "logs prob looks like torch.Size([557])\n", + "torch.from_numpy(rewards) looks like torch.Size([557])\n", + "rewards looks like (517,)\n", + "log_probs looks like (517,)\n", + "logs prob looks like torch.Size([517])\n", + "torch.from_numpy(rewards) looks like torch.Size([517])\n", + "rewards looks like (550,)\n", + "log_probs looks like (550,)\n", + "logs prob looks like torch.Size([550])\n", + "torch.from_numpy(rewards) looks like torch.Size([550])\n", + "rewards looks like (690,)\n", + "log_probs looks like (690,)\n", + "logs prob looks like torch.Size([690])\n", + "torch.from_numpy(rewards) looks like torch.Size([690])\n", + "rewards looks like (591,)\n", + "log_probs looks like (591,)\n", + "logs prob looks like torch.Size([591])\n", + "torch.from_numpy(rewards) looks like torch.Size([591])\n", + "rewards looks like (689,)\n", + "log_probs looks like (689,)\n", + "logs prob looks like torch.Size([689])\n", + "torch.from_numpy(rewards) looks like torch.Size([689])\n", + "rewards looks like (1059,)\n", + "log_probs looks like (1059,)\n", + "logs prob looks like torch.Size([1059])\n", + "torch.from_numpy(rewards) looks like torch.Size([1059])\n", + "rewards looks like (619,)\n", + "log_probs looks like (619,)\n", + "logs prob looks like torch.Size([619])\n", + "torch.from_numpy(rewards) looks like torch.Size([619])\n", + "rewards looks like (527,)\n", + "log_probs looks like (527,)\n", + "logs prob looks like torch.Size([527])\n", + "torch.from_numpy(rewards) looks like torch.Size([527])\n", + "rewards looks like (514,)\n", + "log_probs looks like (514,)\n", + "logs prob looks like torch.Size([514])\n", + "torch.from_numpy(rewards) looks like torch.Size([514])\n", + "rewards looks like (655,)\n", + "log_probs looks like (655,)\n", + "logs prob looks like torch.Size([655])\n", + "torch.from_numpy(rewards) looks like torch.Size([655])\n", + "rewards looks like (667,)\n", + "log_probs looks like (667,)\n", + "logs prob looks like torch.Size([667])\n", + "torch.from_numpy(rewards) looks like torch.Size([667])\n", + "rewards looks like (712,)\n", + "log_probs looks like (712,)\n", + "logs prob looks like torch.Size([712])\n", + "torch.from_numpy(rewards) looks like torch.Size([712])\n", + "rewards looks like (636,)\n", + "log_probs looks like (636,)\n", + "logs prob looks like torch.Size([636])\n", + "torch.from_numpy(rewards) looks like torch.Size([636])\n", + "rewards looks like (620,)\n", + "log_probs looks like (620,)\n", + "logs prob looks like torch.Size([620])\n", + "torch.from_numpy(rewards) looks like torch.Size([620])\n", + "rewards looks like (543,)\n", + "log_probs looks like (543,)\n", + "logs prob looks like torch.Size([543])\n", + "torch.from_numpy(rewards) looks like torch.Size([543])\n", + "rewards looks like (586,)\n", + "log_probs looks like (586,)\n", + "logs prob looks like torch.Size([586])\n", + "torch.from_numpy(rewards) looks like torch.Size([586])\n", + "rewards looks like (498,)\n", + "log_probs looks like (498,)\n", + "logs prob looks like torch.Size([498])\n", + "torch.from_numpy(rewards) looks like torch.Size([498])\n", + "rewards looks like (586,)\n", + "log_probs looks like (586,)\n", + "logs prob looks like torch.Size([586])\n", + "torch.from_numpy(rewards) looks like torch.Size([586])\n", + "rewards looks like (591,)\n", + "log_probs looks like (591,)\n", + "logs prob looks like torch.Size([591])\n", + "torch.from_numpy(rewards) looks like torch.Size([591])\n", + "rewards looks like (693,)\n", + "log_probs looks like (693,)\n", + "logs prob looks like torch.Size([693])\n", + "torch.from_numpy(rewards) looks like torch.Size([693])\n", + "rewards looks like (648,)\n", + "log_probs looks like (648,)\n", + "logs prob looks like torch.Size([648])\n", + "torch.from_numpy(rewards) looks like torch.Size([648])\n", + "rewards looks like (513,)\n", + "log_probs looks like (513,)\n", + "logs prob looks like torch.Size([513])\n", + "torch.from_numpy(rewards) looks like torch.Size([513])\n", + "rewards looks like (574,)\n", + "log_probs looks like (574,)\n", + "logs prob looks like torch.Size([574])\n", + "torch.from_numpy(rewards) looks like torch.Size([574])\n", + "rewards looks like (718,)\n", + "log_probs looks like (718,)\n", + "logs prob looks like torch.Size([718])\n", + "torch.from_numpy(rewards) looks like torch.Size([718])\n", + "rewards looks like (730,)\n", + "log_probs looks like (730,)\n", + "logs prob looks like torch.Size([730])\n", + "torch.from_numpy(rewards) looks like torch.Size([730])\n", + "rewards looks like (668,)\n", + "log_probs looks like (668,)\n", + "logs prob looks like torch.Size([668])\n", + "torch.from_numpy(rewards) looks like torch.Size([668])\n", + "rewards looks like (754,)\n", + "log_probs looks like (754,)\n", + "logs prob looks like torch.Size([754])\n", + "torch.from_numpy(rewards) looks like torch.Size([754])\n", + "rewards looks like (712,)\n", + "log_probs looks like (712,)\n", + "logs prob looks like torch.Size([712])\n", + "torch.from_numpy(rewards) looks like torch.Size([712])\n", + "rewards looks like (470,)\n", + "log_probs looks like (470,)\n", + "logs prob looks like torch.Size([470])\n", + "torch.from_numpy(rewards) looks like torch.Size([470])\n", + "rewards looks like (665,)\n", + "log_probs looks like (665,)\n", + "logs prob looks like torch.Size([665])\n", + "torch.from_numpy(rewards) looks like torch.Size([665])\n", + "rewards looks like (585,)\n", + "log_probs looks like (585,)\n", + "logs prob looks like torch.Size([585])\n", + "torch.from_numpy(rewards) looks like torch.Size([585])\n", + "rewards looks like (512,)\n", + "log_probs looks like (512,)\n", + "logs prob looks like torch.Size([512])\n", + "torch.from_numpy(rewards) looks like torch.Size([512])\n", + "rewards looks like (702,)\n", + "log_probs looks like (702,)\n", + "logs prob looks like torch.Size([702])\n", + "torch.from_numpy(rewards) looks like torch.Size([702])\n", + "rewards looks like (596,)\n", + "log_probs looks like (596,)\n", + "logs prob looks like torch.Size([596])\n", + "torch.from_numpy(rewards) looks like torch.Size([596])\n", + "rewards looks like (626,)\n", + "log_probs looks like (626,)\n", + "logs prob looks like torch.Size([626])\n", + "torch.from_numpy(rewards) looks like torch.Size([626])\n", + "rewards looks like (566,)\n", + "log_probs looks like (566,)\n", + "logs prob looks like torch.Size([566])\n", + "torch.from_numpy(rewards) looks like torch.Size([566])\n", + "rewards looks like (717,)\n", + "log_probs looks like (717,)\n", + "logs prob looks like torch.Size([717])\n", + "torch.from_numpy(rewards) looks like torch.Size([717])\n", + "rewards looks like (708,)\n", + "log_probs looks like (708,)\n", + "logs prob looks like torch.Size([708])\n", + "torch.from_numpy(rewards) looks like torch.Size([708])\n", + "rewards looks like (565,)\n", + "log_probs looks like (565,)\n", + "logs prob looks like torch.Size([565])\n", + "torch.from_numpy(rewards) looks like torch.Size([565])\n", + "rewards looks like (450,)\n", + "log_probs looks like (450,)\n", + "logs prob looks like torch.Size([450])\n", + "torch.from_numpy(rewards) looks like torch.Size([450])\n", + "rewards looks like (584,)\n", + "log_probs looks like (584,)\n", + "logs prob looks like torch.Size([584])\n", + "torch.from_numpy(rewards) looks like torch.Size([584])\n", + "rewards looks like (670,)\n", + "log_probs looks like (670,)\n", + "logs prob looks like torch.Size([670])\n", + "torch.from_numpy(rewards) looks like torch.Size([670])\n", + "rewards looks like (691,)\n", + "log_probs looks like (691,)\n", + "logs prob looks like torch.Size([691])\n", + "torch.from_numpy(rewards) looks like torch.Size([691])\n", + "rewards looks like (760,)\n", + "log_probs looks like (760,)\n", + "logs prob looks like torch.Size([760])\n", + "torch.from_numpy(rewards) looks like torch.Size([760])\n", + "rewards looks like (752,)\n", + "log_probs looks like (752,)\n", + "logs prob looks like torch.Size([752])\n", + "torch.from_numpy(rewards) looks like torch.Size([752])\n", + "rewards looks like (478,)\n", + "log_probs looks like (478,)\n", + "logs prob looks like torch.Size([478])\n", + "torch.from_numpy(rewards) looks like torch.Size([478])\n", + "rewards looks like (553,)\n", + "log_probs looks like (553,)\n", + "logs prob looks like torch.Size([553])\n", + "torch.from_numpy(rewards) looks like torch.Size([553])\n", + "rewards looks like (1660,)\n", + "log_probs looks like (1660,)\n", + "logs prob looks like torch.Size([1660])\n", + "torch.from_numpy(rewards) looks like torch.Size([1660])\n", + "rewards looks like (751,)\n", + "log_probs looks like (751,)\n", + "logs prob looks like torch.Size([751])\n", + "torch.from_numpy(rewards) looks like torch.Size([751])\n", + "rewards looks like (801,)\n", + "log_probs looks like (801,)\n", + "logs prob looks like torch.Size([801])\n", + "torch.from_numpy(rewards) looks like torch.Size([801])\n", + "rewards looks like (715,)\n", + "log_probs looks like (715,)\n", + "logs prob looks like torch.Size([715])\n", + "torch.from_numpy(rewards) looks like torch.Size([715])\n", + "rewards looks like (708,)\n", + "log_probs looks like (708,)\n", + "logs prob looks like torch.Size([708])\n", + "torch.from_numpy(rewards) looks like torch.Size([708])\n", + "rewards looks like (609,)\n", + "log_probs looks like (609,)\n", + "logs prob looks like torch.Size([609])\n", + "torch.from_numpy(rewards) looks like torch.Size([609])\n", + "rewards looks like (732,)\n", + "log_probs looks like (732,)\n", + "logs prob looks like torch.Size([732])\n", + "torch.from_numpy(rewards) looks like torch.Size([732])\n", + "rewards looks like (603,)\n", + "log_probs looks like (603,)\n", + "logs prob looks like torch.Size([603])\n", + "torch.from_numpy(rewards) looks like torch.Size([603])\n", + "rewards looks like (603,)\n", + "log_probs looks like (603,)\n", + "logs prob looks like torch.Size([603])\n", + "torch.from_numpy(rewards) looks like torch.Size([603])\n", + "rewards looks like (665,)\n", + "log_probs looks like (665,)\n", + "logs prob looks like torch.Size([665])\n", + "torch.from_numpy(rewards) looks like torch.Size([665])\n", + "rewards looks like (658,)\n", + "log_probs looks like (658,)\n", + "logs prob looks like torch.Size([658])\n", + "torch.from_numpy(rewards) looks like torch.Size([658])\n", + "rewards looks like (783,)\n", + "log_probs looks like (783,)\n", + "logs prob looks like torch.Size([783])\n", + "torch.from_numpy(rewards) looks like torch.Size([783])\n", + "rewards looks like (652,)\n", + "log_probs looks like (652,)\n", + "logs prob looks like torch.Size([652])\n", + "torch.from_numpy(rewards) looks like torch.Size([652])\n", + "rewards looks like (892,)\n", + "log_probs looks like (892,)\n", + "logs prob looks like torch.Size([892])\n", + "torch.from_numpy(rewards) looks like torch.Size([892])\n", + "rewards looks like (821,)\n", + "log_probs looks like (821,)\n", + "logs prob looks like torch.Size([821])\n", + "torch.from_numpy(rewards) looks like torch.Size([821])\n", + "rewards looks like (986,)\n", + "log_probs looks like (986,)\n", + "logs prob looks like torch.Size([986])\n", + "torch.from_numpy(rewards) looks like torch.Size([986])\n", + "rewards looks like (916,)\n", + "log_probs looks like (916,)\n", + "logs prob looks like torch.Size([916])\n", + "torch.from_numpy(rewards) looks like torch.Size([916])\n", + "rewards looks like (742,)\n", + "log_probs looks like (742,)\n", + "logs prob looks like torch.Size([742])\n", + "torch.from_numpy(rewards) looks like torch.Size([742])\n", + "rewards looks like (604,)\n", + "log_probs looks like (604,)\n", + "logs prob looks like torch.Size([604])\n", + "torch.from_numpy(rewards) looks like torch.Size([604])\n", + "rewards looks like (818,)\n", + "log_probs looks like (818,)\n", + "logs prob looks like torch.Size([818])\n", + "torch.from_numpy(rewards) looks like torch.Size([818])\n", + "rewards looks like (855,)\n", + "log_probs looks like (855,)\n", + "logs prob looks like torch.Size([855])\n", + "torch.from_numpy(rewards) looks like torch.Size([855])\n", + "rewards looks like (795,)\n", + "log_probs looks like (795,)\n", + "logs prob looks like torch.Size([795])\n", + "torch.from_numpy(rewards) looks like torch.Size([795])\n", + "rewards looks like (868,)\n", + "log_probs looks like (868,)\n", + "logs prob looks like torch.Size([868])\n", + "torch.from_numpy(rewards) looks like torch.Size([868])\n", + "rewards looks like (800,)\n", + "log_probs looks like (800,)\n", + "logs prob looks like torch.Size([800])\n", + "torch.from_numpy(rewards) looks like torch.Size([800])\n", + "rewards looks like (820,)\n", + "log_probs looks like (820,)\n", + "logs prob looks like torch.Size([820])\n", + "torch.from_numpy(rewards) looks like torch.Size([820])\n", + "rewards looks like (760,)\n", + "log_probs looks like (760,)\n", + "logs prob looks like torch.Size([760])\n", + "torch.from_numpy(rewards) looks like torch.Size([760])\n", + "rewards looks like (886,)\n", + "log_probs looks like (886,)\n", + "logs prob looks like torch.Size([886])\n", + "torch.from_numpy(rewards) looks like torch.Size([886])\n", + "rewards looks like (1027,)\n", + "log_probs looks like (1027,)\n", + "logs prob looks like torch.Size([1027])\n", + "torch.from_numpy(rewards) looks like torch.Size([1027])\n", + "rewards looks like (819,)\n", + "log_probs looks like (819,)\n", + "logs prob looks like torch.Size([819])\n", + "torch.from_numpy(rewards) looks like torch.Size([819])\n", + "rewards looks like (934,)\n", + "log_probs looks like (934,)\n", + "logs prob looks like torch.Size([934])\n", + "torch.from_numpy(rewards) looks like torch.Size([934])\n", + "rewards looks like (1648,)\n", + "log_probs looks like (1648,)\n", + "logs prob looks like torch.Size([1648])\n", + "torch.from_numpy(rewards) looks like torch.Size([1648])\n", + "rewards looks like (1057,)\n", + "log_probs looks like (1057,)\n", + "logs prob looks like torch.Size([1057])\n", + "torch.from_numpy(rewards) looks like torch.Size([1057])\n", + "rewards looks like (861,)\n", + "log_probs looks like (861,)\n", + "logs prob looks like torch.Size([861])\n", + "torch.from_numpy(rewards) looks like torch.Size([861])\n", + "rewards looks like (1533,)\n", + "log_probs looks like (1533,)\n", + "logs prob looks like torch.Size([1533])\n", + "torch.from_numpy(rewards) looks like torch.Size([1533])\n", + "rewards looks like (920,)\n", + "log_probs looks like (920,)\n", + "logs prob looks like torch.Size([920])\n", + "torch.from_numpy(rewards) looks like torch.Size([920])\n", + "rewards looks like (905,)\n", + "log_probs looks like (905,)\n", + "logs prob looks like torch.Size([905])\n", + "torch.from_numpy(rewards) looks like torch.Size([905])\n", + "rewards looks like (814,)\n", + "log_probs looks like (814,)\n", + "logs prob looks like torch.Size([814])\n", + "torch.from_numpy(rewards) looks like torch.Size([814])\n", + "rewards looks like (809,)\n", + "log_probs looks like (809,)\n", + "logs prob looks like torch.Size([809])\n", + "torch.from_numpy(rewards) looks like torch.Size([809])\n", + "rewards looks like (873,)\n", + "log_probs looks like (873,)\n", + "logs prob looks like torch.Size([873])\n", + "torch.from_numpy(rewards) looks like torch.Size([873])\n", + "rewards looks like (727,)\n", + "log_probs looks like (727,)\n", + "logs prob looks like torch.Size([727])\n", + "torch.from_numpy(rewards) looks like torch.Size([727])\n", + "rewards looks like (1129,)\n", + "log_probs looks like (1129,)\n", + "logs prob looks like torch.Size([1129])\n", + "torch.from_numpy(rewards) looks like torch.Size([1129])\n", + "rewards looks like (1394,)\n", + "log_probs looks like (1394,)\n", + "logs prob looks like torch.Size([1394])\n", + "torch.from_numpy(rewards) looks like torch.Size([1394])\n", + "rewards looks like (884,)\n", + "log_probs looks like (884,)\n", + "logs prob looks like torch.Size([884])\n", + "torch.from_numpy(rewards) looks like torch.Size([884])\n", + "rewards looks like (1132,)\n", + "log_probs looks like (1132,)\n", + "logs prob looks like torch.Size([1132])\n", + "torch.from_numpy(rewards) looks like torch.Size([1132])\n", + "rewards looks like (1007,)\n", + "log_probs looks like (1007,)\n", + "logs prob looks like torch.Size([1007])\n", + "torch.from_numpy(rewards) looks like torch.Size([1007])\n", + "rewards looks like (711,)\n", + "log_probs looks like (711,)\n", + "logs prob looks like torch.Size([711])\n", + "torch.from_numpy(rewards) looks like torch.Size([711])\n", + "rewards looks like (836,)\n", + "log_probs looks like (836,)\n", + "logs prob looks like torch.Size([836])\n", + "torch.from_numpy(rewards) looks like torch.Size([836])\n", + "rewards looks like (1514,)\n", + "log_probs looks like (1514,)\n", + "logs prob looks like torch.Size([1514])\n", + "torch.from_numpy(rewards) looks like torch.Size([1514])\n", + "rewards looks like (896,)\n", + "log_probs looks like (896,)\n", + "logs prob looks like torch.Size([896])\n", + "torch.from_numpy(rewards) looks like torch.Size([896])\n", + "rewards looks like (912,)\n", + "log_probs looks like (912,)\n", + "logs prob looks like torch.Size([912])\n", + "torch.from_numpy(rewards) looks like torch.Size([912])\n", + "rewards looks like (1478,)\n", + "log_probs looks like (1478,)\n", + "logs prob looks like torch.Size([1478])\n", + "torch.from_numpy(rewards) looks like torch.Size([1478])\n", + "rewards looks like (1279,)\n", + "log_probs looks like (1279,)\n", + "logs prob looks like torch.Size([1279])\n", + "torch.from_numpy(rewards) looks like torch.Size([1279])\n", + "rewards looks like (676,)\n", + "log_probs looks like (676,)\n", + "logs prob looks like torch.Size([676])\n", + "torch.from_numpy(rewards) looks like torch.Size([676])\n", + "rewards looks like (1768,)\n", + "log_probs looks like (1768,)\n", + "logs prob looks like torch.Size([1768])\n", + "torch.from_numpy(rewards) looks like torch.Size([1768])\n", + "rewards looks like (897,)\n", + "log_probs looks like (897,)\n", + "logs prob looks like torch.Size([897])\n", + "torch.from_numpy(rewards) looks like torch.Size([897])\n", + "rewards looks like (1252,)\n", + "log_probs looks like (1252,)\n", + "logs prob looks like torch.Size([1252])\n", + "torch.from_numpy(rewards) looks like torch.Size([1252])\n", + "rewards looks like (995,)\n", + "log_probs looks like (995,)\n", + "logs prob looks like torch.Size([995])\n", + "torch.from_numpy(rewards) looks like torch.Size([995])\n", + "rewards looks like (1075,)\n", + "log_probs looks like (1075,)\n", + "logs prob looks like torch.Size([1075])\n", + "torch.from_numpy(rewards) looks like torch.Size([1075])\n", + "rewards looks like (878,)\n", + "log_probs looks like (878,)\n", + "logs prob looks like torch.Size([878])\n", + "torch.from_numpy(rewards) looks like torch.Size([878])\n", + "rewards looks like (1341,)\n", + "log_probs looks like (1341,)\n", + "logs prob looks like torch.Size([1341])\n", + "torch.from_numpy(rewards) looks like torch.Size([1341])\n", + "rewards looks like (1518,)\n", + "log_probs looks like (1518,)\n", + "logs prob looks like torch.Size([1518])\n", + "torch.from_numpy(rewards) looks like torch.Size([1518])\n", + "rewards looks like (1781,)\n", + "log_probs looks like (1781,)\n", + "logs prob looks like torch.Size([1781])\n", + "torch.from_numpy(rewards) looks like torch.Size([1781])\n", + "rewards looks like (1725,)\n", + "log_probs looks like (1725,)\n", + "logs prob looks like torch.Size([1725])\n", + "torch.from_numpy(rewards) looks like torch.Size([1725])\n", + "rewards looks like (1602,)\n", + "log_probs looks like (1602,)\n", + "logs prob looks like torch.Size([1602])\n", + "torch.from_numpy(rewards) looks like torch.Size([1602])\n", + "rewards looks like (846,)\n", + "log_probs looks like (846,)\n", + "logs prob looks like torch.Size([846])\n", + "torch.from_numpy(rewards) looks like torch.Size([846])\n", + "rewards looks like (1211,)\n", + "log_probs looks like (1211,)\n", + "logs prob looks like torch.Size([1211])\n", + "torch.from_numpy(rewards) looks like torch.Size([1211])\n", + "rewards looks like (3273,)\n", + "log_probs looks like (3273,)\n", + "logs prob looks like torch.Size([3273])\n", + "torch.from_numpy(rewards) looks like torch.Size([3273])\n", + "rewards looks like (744,)\n", + "log_probs looks like (744,)\n", + "logs prob looks like torch.Size([744])\n", + "torch.from_numpy(rewards) looks like torch.Size([744])\n", + "rewards looks like (1751,)\n", + "log_probs looks like (1751,)\n", + "logs prob looks like torch.Size([1751])\n", + "torch.from_numpy(rewards) looks like torch.Size([1751])\n", + "rewards looks like (1244,)\n", + "log_probs looks like (1244,)\n", + "logs prob looks like torch.Size([1244])\n", + "torch.from_numpy(rewards) looks like torch.Size([1244])\n", + "rewards looks like (1313,)\n", + "log_probs looks like (1313,)\n", + "logs prob looks like torch.Size([1313])\n", + "torch.from_numpy(rewards) looks like torch.Size([1313])\n", + "rewards looks like (1993,)\n", + "log_probs looks like (1993,)\n", + "logs prob looks like torch.Size([1993])\n", + "torch.from_numpy(rewards) looks like torch.Size([1993])\n", + "rewards looks like (709,)\n", + "log_probs looks like (709,)\n", + "logs prob looks like torch.Size([709])\n", + "torch.from_numpy(rewards) looks like torch.Size([709])\n", + "rewards looks like (934,)\n", + "log_probs looks like (934,)\n", + "logs prob looks like torch.Size([934])\n", + "torch.from_numpy(rewards) looks like torch.Size([934])\n", + "rewards looks like (1386,)\n", + "log_probs looks like (1386,)\n", + "logs prob looks like torch.Size([1386])\n", + "torch.from_numpy(rewards) looks like torch.Size([1386])\n", + "rewards looks like (635,)\n", + "log_probs looks like (635,)\n", + "logs prob looks like torch.Size([635])\n", + "torch.from_numpy(rewards) looks like torch.Size([635])\n", + "rewards looks like (750,)\n", + "log_probs looks like (750,)\n", + "logs prob looks like torch.Size([750])\n", + "torch.from_numpy(rewards) looks like torch.Size([750])\n", + "rewards looks like (1832,)\n", + "log_probs looks like (1832,)\n", + "logs prob looks like torch.Size([1832])\n", + "torch.from_numpy(rewards) looks like torch.Size([1832])\n", + "rewards looks like (1237,)\n", + "log_probs looks like (1237,)\n", + "logs prob looks like torch.Size([1237])\n", + "torch.from_numpy(rewards) looks like torch.Size([1237])\n", + "rewards looks like (1605,)\n", + "log_probs looks like (1605,)\n", + "logs prob looks like torch.Size([1605])\n", + "torch.from_numpy(rewards) looks like torch.Size([1605])\n", + "rewards looks like (718,)\n", + "log_probs looks like (718,)\n", + "logs prob looks like torch.Size([718])\n", + "torch.from_numpy(rewards) looks like torch.Size([718])\n", + "rewards looks like (966,)\n", + "log_probs looks like (966,)\n", + "logs prob looks like torch.Size([966])\n", + "torch.from_numpy(rewards) looks like torch.Size([966])\n", + "rewards looks like (2696,)\n", + "log_probs looks like (2696,)\n", + "logs prob looks like torch.Size([2696])\n", + "torch.from_numpy(rewards) looks like torch.Size([2696])\n", + "rewards looks like (762,)\n", + "log_probs looks like (762,)\n", + "logs prob looks like torch.Size([762])\n", + "torch.from_numpy(rewards) looks like torch.Size([762])\n", + "rewards looks like (1048,)\n", + "log_probs looks like (1048,)\n", + "logs prob looks like torch.Size([1048])\n", + "torch.from_numpy(rewards) looks like torch.Size([1048])\n", + "rewards looks like (1573,)\n", + "log_probs looks like (1573,)\n", + "logs prob looks like torch.Size([1573])\n", + "torch.from_numpy(rewards) looks like torch.Size([1573])\n", + "rewards looks like (2192,)\n", + "log_probs looks like (2192,)\n", + "logs prob looks like torch.Size([2192])\n", + "torch.from_numpy(rewards) looks like torch.Size([2192])\n", + "rewards looks like (599,)\n", + "log_probs looks like (599,)\n", + "logs prob looks like torch.Size([599])\n", + "torch.from_numpy(rewards) looks like torch.Size([599])\n", + "rewards looks like (758,)\n", + "log_probs looks like (758,)\n", + "logs prob looks like torch.Size([758])\n", + "torch.from_numpy(rewards) looks like torch.Size([758])\n", + "rewards looks like (1955,)\n", + "log_probs looks like (1955,)\n", + "logs prob looks like torch.Size([1955])\n", + "torch.from_numpy(rewards) looks like torch.Size([1955])\n", + "rewards looks like (1770,)\n", + "log_probs looks like (1770,)\n", + "logs prob looks like torch.Size([1770])\n", + "torch.from_numpy(rewards) looks like torch.Size([1770])\n", + "rewards looks like (1343,)\n", + "log_probs looks like (1343,)\n", + "logs prob looks like torch.Size([1343])\n", + "torch.from_numpy(rewards) looks like torch.Size([1343])\n", + "rewards looks like (507,)\n", + "log_probs looks like (507,)\n", + "logs prob looks like torch.Size([507])\n", + "torch.from_numpy(rewards) looks like torch.Size([507])\n", + "rewards looks like (1343,)\n", + "log_probs looks like (1343,)\n", + "logs prob looks like torch.Size([1343])\n", + "torch.from_numpy(rewards) looks like torch.Size([1343])\n", + "rewards looks like (1341,)\n", + "log_probs looks like (1341,)\n", + "logs prob looks like torch.Size([1341])\n", + "torch.from_numpy(rewards) looks like torch.Size([1341])\n", + "rewards looks like (1489,)\n", + "log_probs looks like (1489,)\n", + "logs prob looks like torch.Size([1489])\n", + "torch.from_numpy(rewards) looks like torch.Size([1489])\n", + "rewards looks like (3342,)\n", + "log_probs looks like (3342,)\n", + "logs prob looks like torch.Size([3342])\n", + "torch.from_numpy(rewards) looks like torch.Size([3342])\n", + "rewards looks like (1891,)\n", + "log_probs looks like (1891,)\n", + "logs prob looks like torch.Size([1891])\n", + "torch.from_numpy(rewards) looks like torch.Size([1891])\n", + "rewards looks like (1401,)\n", + "log_probs looks like (1401,)\n", + "logs prob looks like torch.Size([1401])\n", + "torch.from_numpy(rewards) looks like torch.Size([1401])\n", + "rewards looks like (2964,)\n", + "log_probs looks like (2964,)\n", + "logs prob looks like torch.Size([2964])\n", + "torch.from_numpy(rewards) looks like torch.Size([2964])\n", + "rewards looks like (1404,)\n", + "log_probs looks like (1404,)\n", + "logs prob looks like torch.Size([1404])\n", + "torch.from_numpy(rewards) looks like torch.Size([1404])\n", + "rewards looks like (780,)\n", + "log_probs looks like (780,)\n", + "logs prob looks like torch.Size([780])\n", + "torch.from_numpy(rewards) looks like torch.Size([780])\n", + "rewards looks like (1632,)\n", + "log_probs looks like (1632,)\n", + "logs prob looks like torch.Size([1632])\n", + "torch.from_numpy(rewards) looks like torch.Size([1632])\n", + "rewards looks like (1578,)\n", + "log_probs looks like (1578,)\n", + "logs prob looks like torch.Size([1578])\n", + "torch.from_numpy(rewards) looks like torch.Size([1578])\n", + "rewards looks like (1082,)\n", + "log_probs looks like (1082,)\n", + "logs prob looks like torch.Size([1082])\n", + "torch.from_numpy(rewards) looks like torch.Size([1082])\n", + "rewards looks like (1423,)\n", + "log_probs looks like (1423,)\n", + "logs prob looks like torch.Size([1423])\n", + "torch.from_numpy(rewards) looks like torch.Size([1423])\n", + "rewards looks like (2867,)\n", + "log_probs looks like (2867,)\n", + "logs prob looks like torch.Size([2867])\n", + "torch.from_numpy(rewards) looks like torch.Size([2867])\n", + "rewards looks like (1733,)\n", + "log_probs looks like (1733,)\n", + "logs prob looks like torch.Size([1733])\n", + "torch.from_numpy(rewards) looks like torch.Size([1733])\n", + "rewards looks like (646,)\n", + "log_probs looks like (646,)\n", + "logs prob looks like torch.Size([646])\n", + "torch.from_numpy(rewards) looks like torch.Size([646])\n", + "rewards looks like (1576,)\n", + "log_probs looks like (1576,)\n", + "logs prob looks like torch.Size([1576])\n", + "torch.from_numpy(rewards) looks like torch.Size([1576])\n", + "rewards looks like (1869,)\n", + "log_probs looks like (1869,)\n", + "logs prob looks like torch.Size([1869])\n", + "torch.from_numpy(rewards) looks like torch.Size([1869])\n", + "rewards looks like (1862,)\n", + "log_probs looks like (1862,)\n", + "logs prob looks like torch.Size([1862])\n", + "torch.from_numpy(rewards) looks like torch.Size([1862])\n", + "rewards looks like (3182,)\n", + "log_probs looks like (3182,)\n", + "logs prob looks like torch.Size([3182])\n", + "torch.from_numpy(rewards) looks like torch.Size([3182])\n", + "rewards looks like (1746,)\n", + "log_probs looks like (1746,)\n", + "logs prob looks like torch.Size([1746])\n", + "torch.from_numpy(rewards) looks like torch.Size([1746])\n", + "rewards looks like (1855,)\n", + "log_probs looks like (1855,)\n", + "logs prob looks like torch.Size([1855])\n", + "torch.from_numpy(rewards) looks like torch.Size([1855])\n", + "rewards looks like (2710,)\n", + "log_probs looks like (2710,)\n", + "logs prob looks like torch.Size([2710])\n", + "torch.from_numpy(rewards) looks like torch.Size([2710])\n", + "rewards looks like (1707,)\n", + "log_probs looks like (1707,)\n", + "logs prob looks like torch.Size([1707])\n", + "torch.from_numpy(rewards) looks like torch.Size([1707])\n", + "rewards looks like (1723,)\n", + "log_probs looks like (1723,)\n", + "logs prob looks like torch.Size([1723])\n", + "torch.from_numpy(rewards) looks like torch.Size([1723])\n", + "rewards looks like (1590,)\n", + "log_probs looks like (1590,)\n", + "logs prob looks like torch.Size([1590])\n", + "torch.from_numpy(rewards) looks like torch.Size([1590])\n", + "rewards looks like (1432,)\n", + "log_probs looks like (1432,)\n", + "logs prob looks like torch.Size([1432])\n", + "torch.from_numpy(rewards) looks like torch.Size([1432])\n", + "rewards looks like (2742,)\n", + "log_probs looks like (2742,)\n", + "logs prob looks like torch.Size([2742])\n", + "torch.from_numpy(rewards) looks like torch.Size([2742])\n", + "rewards looks like (3007,)\n", + "log_probs looks like (3007,)\n", + "logs prob looks like torch.Size([3007])\n", + "torch.from_numpy(rewards) looks like torch.Size([3007])\n", + "rewards looks like (2064,)\n", + "log_probs looks like (2064,)\n", + "logs prob looks like torch.Size([2064])\n", + "torch.from_numpy(rewards) looks like torch.Size([2064])\n", + "rewards looks like (1447,)\n", + "log_probs looks like (1447,)\n", + "logs prob looks like torch.Size([1447])\n", + "torch.from_numpy(rewards) looks like torch.Size([1447])\n", + "rewards looks like (4007,)\n", + "log_probs looks like (4007,)\n", + "logs prob looks like torch.Size([4007])\n", + "torch.from_numpy(rewards) looks like torch.Size([4007])\n", + "rewards looks like (611,)\n", + "log_probs looks like (611,)\n", + "logs prob looks like torch.Size([611])\n", + "torch.from_numpy(rewards) looks like torch.Size([611])\n", + "rewards looks like (1633,)\n", + "log_probs looks like (1633,)\n", + "logs prob looks like torch.Size([1633])\n", + "torch.from_numpy(rewards) looks like torch.Size([1633])\n", + "rewards looks like (3295,)\n", + "log_probs looks like (3295,)\n", + "logs prob looks like torch.Size([3295])\n", + "torch.from_numpy(rewards) looks like torch.Size([3295])\n", + "rewards looks like (975,)\n", + "log_probs looks like (975,)\n", + "logs prob looks like torch.Size([975])\n", + "torch.from_numpy(rewards) looks like torch.Size([975])\n", + "rewards looks like (1991,)\n", + "log_probs looks like (1991,)\n", + "logs prob looks like torch.Size([1991])\n", + "torch.from_numpy(rewards) looks like torch.Size([1991])\n", + "rewards looks like (2409,)\n", + "log_probs looks like (2409,)\n", + "logs prob looks like torch.Size([2409])\n", + "torch.from_numpy(rewards) looks like torch.Size([2409])\n", + "rewards looks like (1587,)\n", + "log_probs looks like (1587,)\n", + "logs prob looks like torch.Size([1587])\n", + "torch.from_numpy(rewards) looks like torch.Size([1587])\n", + "rewards looks like (1334,)\n", + "log_probs looks like (1334,)\n", + "logs prob looks like torch.Size([1334])\n", + "torch.from_numpy(rewards) looks like torch.Size([1334])\n", + "rewards looks like (1070,)\n", + "log_probs looks like (1070,)\n", + "logs prob looks like torch.Size([1070])\n", + "torch.from_numpy(rewards) looks like torch.Size([1070])\n", + "rewards looks like (1082,)\n", + "log_probs looks like (1082,)\n", + "logs prob looks like torch.Size([1082])\n", + "torch.from_numpy(rewards) looks like torch.Size([1082])\n", + "rewards looks like (1084,)\n", + "log_probs looks like (1084,)\n", + "logs prob looks like torch.Size([1084])\n", + "torch.from_numpy(rewards) looks like torch.Size([1084])\n", + "rewards looks like (1192,)\n", + "log_probs looks like (1192,)\n", + "logs prob looks like torch.Size([1192])\n", + "torch.from_numpy(rewards) looks like torch.Size([1192])\n", + "rewards looks like (1287,)\n", + "log_probs looks like (1287,)\n", + "logs prob looks like torch.Size([1287])\n", + "torch.from_numpy(rewards) looks like torch.Size([1287])\n", + "rewards looks like (1718,)\n", + "log_probs looks like (1718,)\n", + "logs prob looks like torch.Size([1718])\n", + "torch.from_numpy(rewards) looks like torch.Size([1718])\n", + "rewards looks like (1859,)\n", + "log_probs looks like (1859,)\n", + "logs prob looks like torch.Size([1859])\n", + "torch.from_numpy(rewards) looks like torch.Size([1859])\n", + "rewards looks like (1215,)\n", + "log_probs looks like (1215,)\n", + "logs prob looks like torch.Size([1215])\n", + "torch.from_numpy(rewards) looks like torch.Size([1215])\n", + "rewards looks like (1181,)\n", + "log_probs looks like (1181,)\n", + "logs prob looks like torch.Size([1181])\n", + "torch.from_numpy(rewards) looks like torch.Size([1181])\n", + "rewards looks like (1378,)\n", + "log_probs looks like (1378,)\n", + "logs prob looks like torch.Size([1378])\n", + "torch.from_numpy(rewards) looks like torch.Size([1378])\n", + "rewards looks like (1851,)\n", + "log_probs looks like (1851,)\n", + "logs prob looks like torch.Size([1851])\n", + "torch.from_numpy(rewards) looks like torch.Size([1851])\n", + "rewards looks like (2218,)\n", + "log_probs looks like (2218,)\n", + "logs prob looks like torch.Size([2218])\n", + "torch.from_numpy(rewards) looks like torch.Size([2218])\n", + "rewards looks like (2502,)\n", + "log_probs looks like (2502,)\n", + "logs prob looks like torch.Size([2502])\n", + "torch.from_numpy(rewards) looks like torch.Size([2502])\n", + "rewards looks like (1642,)\n", + "log_probs looks like (1642,)\n", + "logs prob looks like torch.Size([1642])\n", + "torch.from_numpy(rewards) looks like torch.Size([1642])\n", + "rewards looks like (1892,)\n", + "log_probs looks like (1892,)\n", + "logs prob looks like torch.Size([1892])\n", + "torch.from_numpy(rewards) looks like torch.Size([1892])\n", + "rewards looks like (2003,)\n", + "log_probs looks like (2003,)\n", + "logs prob looks like torch.Size([2003])\n", + "torch.from_numpy(rewards) looks like torch.Size([2003])\n", + "rewards looks like (3407,)\n", + "log_probs looks like (3407,)\n", + "logs prob looks like torch.Size([3407])\n", + "torch.from_numpy(rewards) looks like torch.Size([3407])\n", + "rewards looks like (3425,)\n", + "log_probs looks like (3425,)\n", + "logs prob looks like torch.Size([3425])\n", + "torch.from_numpy(rewards) looks like torch.Size([3425])\n", + "rewards looks like (1840,)\n", + "log_probs looks like (1840,)\n", + "logs prob looks like torch.Size([1840])\n", + "torch.from_numpy(rewards) looks like torch.Size([1840])\n", + "rewards looks like (1529,)\n", + "log_probs looks like (1529,)\n", + "logs prob looks like torch.Size([1529])\n", + "torch.from_numpy(rewards) looks like torch.Size([1529])\n", + "rewards looks like (1407,)\n", + "log_probs looks like (1407,)\n", + "logs prob looks like torch.Size([1407])\n", + "torch.from_numpy(rewards) looks like torch.Size([1407])\n", + "rewards looks like (2541,)\n", + "log_probs looks like (2541,)\n", + "logs prob looks like torch.Size([2541])\n", + "torch.from_numpy(rewards) looks like torch.Size([2541])\n", + "rewards looks like (1194,)\n", + "log_probs looks like (1194,)\n", + "logs prob looks like torch.Size([1194])\n", + "torch.from_numpy(rewards) looks like torch.Size([1194])\n", + "rewards looks like (1431,)\n", + "log_probs looks like (1431,)\n", + "logs prob looks like torch.Size([1431])\n", + "torch.from_numpy(rewards) looks like torch.Size([1431])\n", + "rewards looks like (3340,)\n", + "log_probs looks like (3340,)\n", + "logs prob looks like torch.Size([3340])\n", + "torch.from_numpy(rewards) looks like torch.Size([3340])\n", + "rewards looks like (897,)\n", + "log_probs looks like (897,)\n", + "logs prob looks like torch.Size([897])\n", + "torch.from_numpy(rewards) looks like torch.Size([897])\n", + "rewards looks like (1821,)\n", + "log_probs looks like (1821,)\n", + "logs prob looks like torch.Size([1821])\n", + "torch.from_numpy(rewards) looks like torch.Size([1821])\n", + "rewards looks like (1906,)\n", + "log_probs looks like (1906,)\n", + "logs prob looks like torch.Size([1906])\n", + "torch.from_numpy(rewards) looks like torch.Size([1906])\n", + "rewards looks like (2688,)\n", + "log_probs looks like (2688,)\n", + "logs prob looks like torch.Size([2688])\n", + "torch.from_numpy(rewards) looks like torch.Size([2688])\n", + "rewards looks like (1169,)\n", + "log_probs looks like (1169,)\n", + "logs prob looks like torch.Size([1169])\n", + "torch.from_numpy(rewards) looks like torch.Size([1169])\n", + "rewards looks like (1444,)\n", + "log_probs looks like (1444,)\n", + "logs prob looks like torch.Size([1444])\n", + "torch.from_numpy(rewards) looks like torch.Size([1444])\n", + "rewards looks like (1376,)\n", + "log_probs looks like (1376,)\n", + "logs prob looks like torch.Size([1376])\n", + "torch.from_numpy(rewards) looks like torch.Size([1376])\n", + "rewards looks like (1395,)\n", + "log_probs looks like (1395,)\n", + "logs prob looks like torch.Size([1395])\n", + "torch.from_numpy(rewards) looks like torch.Size([1395])\n", + "rewards looks like (899,)\n", + "log_probs looks like (899,)\n", + "logs prob looks like torch.Size([899])\n", + "torch.from_numpy(rewards) looks like torch.Size([899])\n", + "rewards looks like (2152,)\n", + "log_probs looks like (2152,)\n", + "logs prob looks like torch.Size([2152])\n", + "torch.from_numpy(rewards) looks like torch.Size([2152])\n", + "rewards looks like (2294,)\n", + "log_probs looks like (2294,)\n", + "logs prob looks like torch.Size([2294])\n", + "torch.from_numpy(rewards) looks like torch.Size([2294])\n", + "rewards looks like (881,)\n", + "log_probs looks like (881,)\n", + "logs prob looks like torch.Size([881])\n", + "torch.from_numpy(rewards) looks like torch.Size([881])\n", + "rewards looks like (1050,)\n", + "log_probs looks like (1050,)\n", + "logs prob looks like torch.Size([1050])\n", + "torch.from_numpy(rewards) looks like torch.Size([1050])\n", + "rewards looks like (1294,)\n", + "log_probs looks like (1294,)\n", + "logs prob looks like torch.Size([1294])\n", + "torch.from_numpy(rewards) looks like torch.Size([1294])\n", + "rewards looks like (1419,)\n", + "log_probs looks like (1419,)\n", + "logs prob looks like torch.Size([1419])\n", + "torch.from_numpy(rewards) looks like torch.Size([1419])\n", + "rewards looks like (1433,)\n", + "log_probs looks like (1433,)\n", + "logs prob looks like torch.Size([1433])\n", + "torch.from_numpy(rewards) looks like torch.Size([1433])\n", + "rewards looks like (2196,)\n", + "log_probs looks like (2196,)\n", + "logs prob looks like torch.Size([2196])\n", + "torch.from_numpy(rewards) looks like torch.Size([2196])\n", + "rewards looks like (1811,)\n", + "log_probs looks like (1811,)\n", + "logs prob looks like torch.Size([1811])\n", + "torch.from_numpy(rewards) looks like torch.Size([1811])\n", + "rewards looks like (1418,)\n", + "log_probs looks like (1418,)\n", + "logs prob looks like torch.Size([1418])\n", + "torch.from_numpy(rewards) looks like torch.Size([1418])\n", + "rewards looks like (1536,)\n", + "log_probs looks like (1536,)\n", + "logs prob looks like torch.Size([1536])\n", + "torch.from_numpy(rewards) looks like torch.Size([1536])\n", + "rewards looks like (1353,)\n", + "log_probs looks like (1353,)\n", + "logs prob looks like torch.Size([1353])\n", + "torch.from_numpy(rewards) looks like torch.Size([1353])\n", + "rewards looks like (1260,)\n", + "log_probs looks like (1260,)\n", + "logs prob looks like torch.Size([1260])\n", + "torch.from_numpy(rewards) looks like torch.Size([1260])\n", + "rewards looks like (1514,)\n", + "log_probs looks like (1514,)\n", + "logs prob looks like torch.Size([1514])\n", + "torch.from_numpy(rewards) looks like torch.Size([1514])\n", + "rewards looks like (2095,)\n", + "log_probs looks like (2095,)\n", + "logs prob looks like torch.Size([2095])\n", + "torch.from_numpy(rewards) looks like torch.Size([2095])\n", + "rewards looks like (1695,)\n", + "log_probs looks like (1695,)\n", + "logs prob looks like torch.Size([1695])\n", + "torch.from_numpy(rewards) looks like torch.Size([1695])\n", + "rewards looks like (2109,)\n", + "log_probs looks like (2109,)\n", + "logs prob looks like torch.Size([2109])\n", + "torch.from_numpy(rewards) looks like torch.Size([2109])\n", + "rewards looks like (967,)\n", + "log_probs looks like (967,)\n", + "logs prob looks like torch.Size([967])\n", + "torch.from_numpy(rewards) looks like torch.Size([967])\n", + "rewards looks like (1231,)\n", + "log_probs looks like (1231,)\n", + "logs prob looks like torch.Size([1231])\n", + "torch.from_numpy(rewards) looks like torch.Size([1231])\n", + "rewards looks like (1355,)\n", + "log_probs looks like (1355,)\n", + "logs prob looks like torch.Size([1355])\n", + "torch.from_numpy(rewards) looks like torch.Size([1355])\n", + "rewards looks like (1351,)\n", + "log_probs looks like (1351,)\n", + "logs prob looks like torch.Size([1351])\n", + "torch.from_numpy(rewards) looks like torch.Size([1351])\n", + "rewards looks like (1674,)\n", + "log_probs looks like (1674,)\n", + "logs prob looks like torch.Size([1674])\n", + "torch.from_numpy(rewards) looks like torch.Size([1674])\n", + "rewards looks like (2394,)\n", + "log_probs looks like (2394,)\n", + "logs prob looks like torch.Size([2394])\n", + "torch.from_numpy(rewards) looks like torch.Size([2394])\n", + "rewards looks like (2296,)\n", + "log_probs looks like (2296,)\n", + "logs prob looks like torch.Size([2296])\n", + "torch.from_numpy(rewards) looks like torch.Size([2296])\n", + "rewards looks like (897,)\n", + "log_probs looks like (897,)\n", + "logs prob looks like torch.Size([897])\n", + "torch.from_numpy(rewards) looks like torch.Size([897])\n", + "rewards looks like (2389,)\n", + "log_probs looks like (2389,)\n", + "logs prob looks like torch.Size([2389])\n", + "torch.from_numpy(rewards) looks like torch.Size([2389])\n", + "rewards looks like (1798,)\n", + "log_probs looks like (1798,)\n", + "logs prob looks like torch.Size([1798])\n", + "torch.from_numpy(rewards) looks like torch.Size([1798])\n", + "rewards looks like (1232,)\n", + "log_probs looks like (1232,)\n", + "logs prob looks like torch.Size([1232])\n", + "torch.from_numpy(rewards) looks like torch.Size([1232])\n", + "rewards looks like (1173,)\n", + "log_probs looks like (1173,)\n", + "logs prob looks like torch.Size([1173])\n", + "torch.from_numpy(rewards) looks like torch.Size([1173])\n", + "rewards looks like (1165,)\n", + "log_probs looks like (1165,)\n", + "logs prob looks like torch.Size([1165])\n", + "torch.from_numpy(rewards) looks like torch.Size([1165])\n", + "rewards looks like (1602,)\n", + "log_probs looks like (1602,)\n", + "logs prob looks like torch.Size([1602])\n", + "torch.from_numpy(rewards) looks like torch.Size([1602])\n", + "rewards looks like (1164,)\n", + "log_probs looks like (1164,)\n", + "logs prob looks like torch.Size([1164])\n", + "torch.from_numpy(rewards) looks like torch.Size([1164])\n", + "rewards looks like (2235,)\n", + "log_probs looks like (2235,)\n", + "logs prob looks like torch.Size([2235])\n", + "torch.from_numpy(rewards) looks like torch.Size([2235])\n", + "rewards looks like (1038,)\n", + "log_probs looks like (1038,)\n", + "logs prob looks like torch.Size([1038])\n", + "torch.from_numpy(rewards) looks like torch.Size([1038])\n", + "rewards looks like (1698,)\n", + "log_probs looks like (1698,)\n", + "logs prob looks like torch.Size([1698])\n", + "torch.from_numpy(rewards) looks like torch.Size([1698])\n", + "rewards looks like (1436,)\n", + "log_probs looks like (1436,)\n", + "logs prob looks like torch.Size([1436])\n", + "torch.from_numpy(rewards) looks like torch.Size([1436])\n", + "rewards looks like (1223,)\n", + "log_probs looks like (1223,)\n", + "logs prob looks like torch.Size([1223])\n", + "torch.from_numpy(rewards) looks like torch.Size([1223])\n", + "rewards looks like (2006,)\n", + "log_probs looks like (2006,)\n", + "logs prob looks like torch.Size([2006])\n", + "torch.from_numpy(rewards) looks like torch.Size([2006])\n", + "rewards looks like (1162,)\n", + "log_probs looks like (1162,)\n", + "logs prob looks like torch.Size([1162])\n", + "torch.from_numpy(rewards) looks like torch.Size([1162])\n", + "rewards looks like (2239,)\n", + "log_probs looks like (2239,)\n", + "logs prob looks like torch.Size([2239])\n", + "torch.from_numpy(rewards) looks like torch.Size([2239])\n", + "rewards looks like (1104,)\n", + "log_probs looks like (1104,)\n", + "logs prob looks like torch.Size([1104])\n", + "torch.from_numpy(rewards) looks like torch.Size([1104])\n", + "rewards looks like (1389,)\n", + "log_probs looks like (1389,)\n", + "logs prob looks like torch.Size([1389])\n", + "torch.from_numpy(rewards) looks like torch.Size([1389])\n", + "rewards looks like (1419,)\n", + "log_probs looks like (1419,)\n", + "logs prob looks like torch.Size([1419])\n", + "torch.from_numpy(rewards) looks like torch.Size([1419])\n", + "rewards looks like (1526,)\n", + "log_probs looks like (1526,)\n", + "logs prob looks like torch.Size([1526])\n", + "torch.from_numpy(rewards) looks like torch.Size([1526])\n", + "rewards looks like (1618,)\n", + "log_probs looks like (1618,)\n", + "logs prob looks like torch.Size([1618])\n", + "torch.from_numpy(rewards) looks like torch.Size([1618])\n", + "rewards looks like (2276,)\n", + "log_probs looks like (2276,)\n", + "logs prob looks like torch.Size([2276])\n", + "torch.from_numpy(rewards) looks like torch.Size([2276])\n", + "rewards looks like (2973,)\n", + "log_probs looks like (2973,)\n", + "logs prob looks like torch.Size([2973])\n", + "torch.from_numpy(rewards) looks like torch.Size([2973])\n", + "rewards looks like (1418,)\n", + "log_probs looks like (1418,)\n", + "logs prob looks like torch.Size([1418])\n", + "torch.from_numpy(rewards) looks like torch.Size([1418])\n", + "rewards looks like (1273,)\n", + "log_probs looks like (1273,)\n", + "logs prob looks like torch.Size([1273])\n", + "torch.from_numpy(rewards) looks like torch.Size([1273])\n", + "rewards looks like (2355,)\n", + "log_probs looks like (2355,)\n", + "logs prob looks like torch.Size([2355])\n", + "torch.from_numpy(rewards) looks like torch.Size([2355])\n", + "rewards looks like (1308,)\n", + "log_probs looks like (1308,)\n", + "logs prob looks like torch.Size([1308])\n", + "torch.from_numpy(rewards) looks like torch.Size([1308])\n", + "rewards looks like (1403,)\n", + "log_probs looks like (1403,)\n", + "logs prob looks like torch.Size([1403])\n", + "torch.from_numpy(rewards) looks like torch.Size([1403])\n", + "rewards looks like (1794,)\n", + "log_probs looks like (1794,)\n", + "logs prob looks like torch.Size([1794])\n", + "torch.from_numpy(rewards) looks like torch.Size([1794])\n", + "rewards looks like (1101,)\n", + "log_probs looks like (1101,)\n", + "logs prob looks like torch.Size([1101])\n", + "torch.from_numpy(rewards) looks like torch.Size([1101])\n", + "rewards looks like (1165,)\n", + "log_probs looks like (1165,)\n", + "logs prob looks like torch.Size([1165])\n", + "torch.from_numpy(rewards) looks like torch.Size([1165])\n", + "rewards looks like (1162,)\n", + "log_probs looks like (1162,)\n", + "logs prob looks like torch.Size([1162])\n", + "torch.from_numpy(rewards) looks like torch.Size([1162])\n", + "rewards looks like (1317,)\n", + "log_probs looks like (1317,)\n", + "logs prob looks like torch.Size([1317])\n", + "torch.from_numpy(rewards) looks like torch.Size([1317])\n", + "rewards looks like (993,)\n", + "log_probs looks like (993,)\n", + "logs prob looks like torch.Size([993])\n", + "torch.from_numpy(rewards) looks like torch.Size([993])\n", + "rewards looks like (2078,)\n", + "log_probs looks like (2078,)\n", + "logs prob looks like torch.Size([2078])\n", + "torch.from_numpy(rewards) looks like torch.Size([2078])\n", + "rewards looks like (1419,)\n", + "log_probs looks like (1419,)\n", + "logs prob looks like torch.Size([1419])\n", + "torch.from_numpy(rewards) looks like torch.Size([1419])\n", + "rewards looks like (1354,)\n", + "log_probs looks like (1354,)\n", + "logs prob looks like torch.Size([1354])\n", + "torch.from_numpy(rewards) looks like torch.Size([1354])\n", + "rewards looks like (1216,)\n", + "log_probs looks like (1216,)\n", + "logs prob looks like torch.Size([1216])\n", + "torch.from_numpy(rewards) looks like torch.Size([1216])\n", + "rewards looks like (1661,)\n", + "log_probs looks like (1661,)\n", + "logs prob looks like torch.Size([1661])\n", + "torch.from_numpy(rewards) looks like torch.Size([1661])\n", + "rewards looks like (2095,)\n", + "log_probs looks like (2095,)\n", + "logs prob looks like torch.Size([2095])\n", + "torch.from_numpy(rewards) looks like torch.Size([2095])\n", + "rewards looks like (2455,)\n", + "log_probs looks like (2455,)\n", + "logs prob looks like torch.Size([2455])\n", + "torch.from_numpy(rewards) looks like torch.Size([2455])\n", + "rewards looks like (2383,)\n", + "log_probs looks like (2383,)\n", + "logs prob looks like torch.Size([2383])\n", + "torch.from_numpy(rewards) looks like torch.Size([2383])\n", + "rewards looks like (2222,)\n", + "log_probs looks like (2222,)\n", + "logs prob looks like torch.Size([2222])\n", + "torch.from_numpy(rewards) looks like torch.Size([2222])\n", + "rewards looks like (2269,)\n", + "log_probs looks like (2269,)\n", + "logs prob looks like torch.Size([2269])\n", + "torch.from_numpy(rewards) looks like torch.Size([2269])\n", + "rewards looks like (2995,)\n", + "log_probs looks like (2995,)\n", + "logs prob looks like torch.Size([2995])\n", + "torch.from_numpy(rewards) looks like torch.Size([2995])\n", + "rewards looks like (1474,)\n", + "log_probs looks like (1474,)\n", + "logs prob looks like torch.Size([1474])\n", + "torch.from_numpy(rewards) looks like torch.Size([1474])\n", + "rewards looks like (2666,)\n", + "log_probs looks like (2666,)\n", + "logs prob looks like torch.Size([2666])\n", + "torch.from_numpy(rewards) looks like torch.Size([2666])\n", + "rewards looks like (1386,)\n", + "log_probs looks like (1386,)\n", + "logs prob looks like torch.Size([1386])\n", + "torch.from_numpy(rewards) looks like torch.Size([1386])\n", + "rewards looks like (2039,)\n", + "log_probs looks like (2039,)\n", + "logs prob looks like torch.Size([2039])\n", + "torch.from_numpy(rewards) looks like torch.Size([2039])\n", + "rewards looks like (2172,)\n", + "log_probs looks like (2172,)\n", + "logs prob looks like torch.Size([2172])\n", + "torch.from_numpy(rewards) looks like torch.Size([2172])\n", + "rewards looks like (2070,)\n", + "log_probs looks like (2070,)\n", + "logs prob looks like torch.Size([2070])\n", + "torch.from_numpy(rewards) looks like torch.Size([2070])\n", + "rewards looks like (2534,)\n", + "log_probs looks like (2534,)\n", + "logs prob looks like torch.Size([2534])\n", + "torch.from_numpy(rewards) looks like torch.Size([2534])\n", + "rewards looks like (1660,)\n", + "log_probs looks like (1660,)\n", + "logs prob looks like torch.Size([1660])\n", + "torch.from_numpy(rewards) looks like torch.Size([1660])\n", + "rewards looks like (1406,)\n", + "log_probs looks like (1406,)\n", + "logs prob looks like torch.Size([1406])\n", + "torch.from_numpy(rewards) looks like torch.Size([1406])\n", + "rewards looks like (1472,)\n", + "log_probs looks like (1472,)\n", + "logs prob looks like torch.Size([1472])\n", + "torch.from_numpy(rewards) looks like torch.Size([1472])\n", + "rewards looks like (2711,)\n", + "log_probs looks like (2711,)\n", + "logs prob looks like torch.Size([2711])\n", + "torch.from_numpy(rewards) looks like torch.Size([2711])\n", + "rewards looks like (1529,)\n", + "log_probs looks like (1529,)\n", + "logs prob looks like torch.Size([1529])\n", + "torch.from_numpy(rewards) looks like torch.Size([1529])\n", + "rewards looks like (1867,)\n", + "log_probs looks like (1867,)\n", + "logs prob looks like torch.Size([1867])\n", + "torch.from_numpy(rewards) looks like torch.Size([1867])\n", + "rewards looks like (1218,)\n", + "log_probs looks like (1218,)\n", + "logs prob looks like torch.Size([1218])\n", + "torch.from_numpy(rewards) looks like torch.Size([1218])\n", + "rewards looks like (1345,)\n", + "log_probs looks like (1345,)\n", + "logs prob looks like torch.Size([1345])\n", + "torch.from_numpy(rewards) looks like torch.Size([1345])\n", + "rewards looks like (1188,)\n", + "log_probs looks like (1188,)\n", + "logs prob looks like torch.Size([1188])\n", + "torch.from_numpy(rewards) looks like torch.Size([1188])\n", + "rewards looks like (1945,)\n", + "log_probs looks like (1945,)\n", + "logs prob looks like torch.Size([1945])\n", + "torch.from_numpy(rewards) looks like torch.Size([1945])\n", + "rewards looks like (987,)\n", + "log_probs looks like (987,)\n", + "logs prob looks like torch.Size([987])\n", + "torch.from_numpy(rewards) looks like torch.Size([987])\n", + "rewards looks like (2017,)\n", + "log_probs looks like (2017,)\n", + "logs prob looks like torch.Size([2017])\n", + "torch.from_numpy(rewards) looks like torch.Size([2017])\n", + "rewards looks like (2001,)\n", + "log_probs looks like (2001,)\n", + "logs prob looks like torch.Size([2001])\n", + "torch.from_numpy(rewards) looks like torch.Size([2001])\n", + "rewards looks like (1335,)\n", + "log_probs looks like (1335,)\n", + "logs prob looks like torch.Size([1335])\n", + "torch.from_numpy(rewards) looks like torch.Size([1335])\n", + "rewards looks like (1343,)\n", + "log_probs looks like (1343,)\n", + "logs prob looks like torch.Size([1343])\n", + "torch.from_numpy(rewards) looks like torch.Size([1343])\n", + "rewards looks like (2834,)\n", + "log_probs looks like (2834,)\n", + "logs prob looks like torch.Size([2834])\n", + "torch.from_numpy(rewards) looks like torch.Size([2834])\n", + "rewards looks like (1391,)\n", + "log_probs looks like (1391,)\n", + "logs prob looks like torch.Size([1391])\n", + "torch.from_numpy(rewards) looks like torch.Size([1391])\n", + "rewards looks like (1852,)\n", + "log_probs looks like (1852,)\n", + "logs prob looks like torch.Size([1852])\n", + "torch.from_numpy(rewards) looks like torch.Size([1852])\n", + "rewards looks like (1256,)\n", + "log_probs looks like (1256,)\n", + "logs prob looks like torch.Size([1256])\n", + "torch.from_numpy(rewards) looks like torch.Size([1256])\n", + "rewards looks like (1184,)\n", + "log_probs looks like (1184,)\n", + "logs prob looks like torch.Size([1184])\n", + "torch.from_numpy(rewards) looks like torch.Size([1184])\n", + "rewards looks like (1939,)\n", + "log_probs looks like (1939,)\n", + "logs prob looks like torch.Size([1939])\n", + "torch.from_numpy(rewards) looks like torch.Size([1939])\n", + "rewards looks like (1274,)\n", + "log_probs looks like (1274,)\n", + "logs prob looks like torch.Size([1274])\n", + "torch.from_numpy(rewards) looks like torch.Size([1274])\n", + "rewards looks like (1367,)\n", + "log_probs looks like (1367,)\n", + "logs prob looks like torch.Size([1367])\n", + "torch.from_numpy(rewards) looks like torch.Size([1367])\n", + "rewards looks like (1284,)\n", + "log_probs looks like (1284,)\n", + "logs prob looks like torch.Size([1284])\n", + "torch.from_numpy(rewards) looks like torch.Size([1284])\n", + "rewards looks like (1127,)\n", + "log_probs looks like (1127,)\n", + "logs prob looks like torch.Size([1127])\n", + "torch.from_numpy(rewards) looks like torch.Size([1127])\n", + "rewards looks like (1298,)\n", + "log_probs looks like (1298,)\n", + "logs prob looks like torch.Size([1298])\n", + "torch.from_numpy(rewards) looks like torch.Size([1298])\n", + "rewards looks like (1638,)\n", + "log_probs looks like (1638,)\n", + "logs prob looks like torch.Size([1638])\n", + "torch.from_numpy(rewards) looks like torch.Size([1638])\n", + "rewards looks like (1144,)\n", + "log_probs looks like (1144,)\n", + "logs prob looks like torch.Size([1144])\n", + "torch.from_numpy(rewards) looks like torch.Size([1144])\n", + "rewards looks like (1370,)\n", + "log_probs looks like (1370,)\n", + "logs prob looks like torch.Size([1370])\n", + "torch.from_numpy(rewards) looks like torch.Size([1370])\n", + "rewards looks like (1835,)\n", + "log_probs looks like (1835,)\n", + "logs prob looks like torch.Size([1835])\n", + "torch.from_numpy(rewards) looks like torch.Size([1835])\n", + "rewards looks like (2149,)\n", + "log_probs looks like (2149,)\n", + "logs prob looks like torch.Size([2149])\n", + "torch.from_numpy(rewards) looks like torch.Size([2149])\n", + "rewards looks like (1033,)\n", + "log_probs looks like (1033,)\n", + "logs prob looks like torch.Size([1033])\n", + "torch.from_numpy(rewards) looks like torch.Size([1033])\n", + "rewards looks like (989,)\n", + "log_probs looks like (989,)\n", + "logs prob looks like torch.Size([989])\n", + "torch.from_numpy(rewards) looks like torch.Size([989])\n", + "rewards looks like (1900,)\n", + "log_probs looks like (1900,)\n", + "logs prob looks like torch.Size([1900])\n", + "torch.from_numpy(rewards) looks like torch.Size([1900])\n", + "rewards looks like (1706,)\n", + "log_probs looks like (1706,)\n", + "logs prob looks like torch.Size([1706])\n", + "torch.from_numpy(rewards) looks like torch.Size([1706])\n", + "rewards looks like (1235,)\n", + "log_probs looks like (1235,)\n", + "logs prob looks like torch.Size([1235])\n", + "torch.from_numpy(rewards) looks like torch.Size([1235])\n", + "rewards looks like (2693,)\n", + "log_probs looks like (2693,)\n", + "logs prob looks like torch.Size([2693])\n", + "torch.from_numpy(rewards) looks like torch.Size([2693])\n", + "rewards looks like (1021,)\n", + "log_probs looks like (1021,)\n", + "logs prob looks like torch.Size([1021])\n", + "torch.from_numpy(rewards) looks like torch.Size([1021])\n", + "rewards looks like (1126,)\n", + "log_probs looks like (1126,)\n", + "logs prob looks like torch.Size([1126])\n", + "torch.from_numpy(rewards) looks like torch.Size([1126])\n", + "rewards looks like (1334,)\n", + "log_probs looks like (1334,)\n", + "logs prob looks like torch.Size([1334])\n", + "torch.from_numpy(rewards) looks like torch.Size([1334])\n", + "rewards looks like (1337,)\n", + "log_probs looks like (1337,)\n", + "logs prob looks like torch.Size([1337])\n", + "torch.from_numpy(rewards) looks like torch.Size([1337])\n", + "rewards looks like (1502,)\n", + "log_probs looks like (1502,)\n", + "logs prob looks like torch.Size([1502])\n", + "torch.from_numpy(rewards) looks like torch.Size([1502])\n", + "rewards looks like (2059,)\n", + "log_probs looks like (2059,)\n", + "logs prob looks like torch.Size([2059])\n", + "torch.from_numpy(rewards) looks like torch.Size([2059])\n", + "rewards looks like (2057,)\n", + "log_probs looks like (2057,)\n", + "logs prob looks like torch.Size([2057])\n", + "torch.from_numpy(rewards) looks like torch.Size([2057])\n", + "rewards looks like (1300,)\n", + "log_probs looks like (1300,)\n", + "logs prob looks like torch.Size([1300])\n", + "torch.from_numpy(rewards) looks like torch.Size([1300])\n", + "rewards looks like (3078,)\n", + "log_probs looks like (3078,)\n", + "logs prob looks like torch.Size([3078])\n", + "torch.from_numpy(rewards) looks like torch.Size([3078])\n", + "rewards looks like (1724,)\n", + "log_probs looks like (1724,)\n", + "logs prob looks like torch.Size([1724])\n", + "torch.from_numpy(rewards) looks like torch.Size([1724])\n", + "rewards looks like (1468,)\n", + "log_probs looks like (1468,)\n", + "logs prob looks like torch.Size([1468])\n", + "torch.from_numpy(rewards) looks like torch.Size([1468])\n", + "rewards looks like (2674,)\n", + "log_probs looks like (2674,)\n", + "logs prob looks like torch.Size([2674])\n", + "torch.from_numpy(rewards) looks like torch.Size([2674])\n", + "rewards looks like (1376,)\n", + "log_probs looks like (1376,)\n", + "logs prob looks like torch.Size([1376])\n", + "torch.from_numpy(rewards) looks like torch.Size([1376])\n", + "rewards looks like (1564,)\n", + "log_probs looks like (1564,)\n", + "logs prob looks like torch.Size([1564])\n", + "torch.from_numpy(rewards) looks like torch.Size([1564])\n", + "rewards looks like (1452,)\n", + "log_probs looks like (1452,)\n", + "logs prob looks like torch.Size([1452])\n", + "torch.from_numpy(rewards) looks like torch.Size([1452])\n", + "rewards looks like (1205,)\n", + "log_probs looks like (1205,)\n", + "logs prob looks like torch.Size([1205])\n", + "torch.from_numpy(rewards) looks like torch.Size([1205])\n", + "rewards looks like (1520,)\n", + "log_probs looks like (1520,)\n", + "logs prob looks like torch.Size([1520])\n", + "torch.from_numpy(rewards) looks like torch.Size([1520])\n", + "rewards looks like (1099,)\n", + "log_probs looks like (1099,)\n", + "logs prob looks like torch.Size([1099])\n", + "torch.from_numpy(rewards) looks like torch.Size([1099])\n", + "rewards looks like (1506,)\n", + "log_probs looks like (1506,)\n", + "logs prob looks like torch.Size([1506])\n", + "torch.from_numpy(rewards) looks like torch.Size([1506])\n", + "rewards looks like (1175,)\n", + "log_probs looks like (1175,)\n", + "logs prob looks like torch.Size([1175])\n", + "torch.from_numpy(rewards) looks like torch.Size([1175])\n", + "rewards looks like (1251,)\n", + "log_probs looks like (1251,)\n", + "logs prob looks like torch.Size([1251])\n", + "torch.from_numpy(rewards) looks like torch.Size([1251])\n", + "rewards looks like (1318,)\n", + "log_probs looks like (1318,)\n", + "logs prob looks like torch.Size([1318])\n", + "torch.from_numpy(rewards) looks like torch.Size([1318])\n", + "rewards looks like (1446,)\n", + "log_probs looks like (1446,)\n", + "logs prob looks like torch.Size([1446])\n", + "torch.from_numpy(rewards) looks like torch.Size([1446])\n", + "rewards looks like (1220,)\n", + "log_probs looks like (1220,)\n", + "logs prob looks like torch.Size([1220])\n", + "torch.from_numpy(rewards) looks like torch.Size([1220])\n", + "rewards looks like (1343,)\n", + "log_probs looks like (1343,)\n", + "logs prob looks like torch.Size([1343])\n", + "torch.from_numpy(rewards) looks like torch.Size([1343])\n", + "rewards looks like (1186,)\n", + "log_probs looks like (1186,)\n", + "logs prob looks like torch.Size([1186])\n", + "torch.from_numpy(rewards) looks like torch.Size([1186])\n", + "rewards looks like (1443,)\n", + "log_probs looks like (1443,)\n", + "logs prob looks like torch.Size([1443])\n", + "torch.from_numpy(rewards) looks like torch.Size([1443])\n", + "rewards looks like (1212,)\n", + "log_probs looks like (1212,)\n", + "logs prob looks like torch.Size([1212])\n", + "torch.from_numpy(rewards) looks like torch.Size([1212])\n", + "rewards looks like (1346,)\n", + "log_probs looks like (1346,)\n", + "logs prob looks like torch.Size([1346])\n", + "torch.from_numpy(rewards) looks like torch.Size([1346])\n", + "rewards looks like (2124,)\n", + "log_probs looks like (2124,)\n", + "logs prob looks like torch.Size([2124])\n", + "torch.from_numpy(rewards) looks like torch.Size([2124])\n", + "rewards looks like (1461,)\n", + "log_probs looks like (1461,)\n", + "logs prob looks like torch.Size([1461])\n", + "torch.from_numpy(rewards) looks like torch.Size([1461])\n", + "rewards looks like (1425,)\n", + "log_probs looks like (1425,)\n", + "logs prob looks like torch.Size([1425])\n", + "torch.from_numpy(rewards) looks like torch.Size([1425])\n", + "rewards looks like (1457,)\n", + "log_probs looks like (1457,)\n", + "logs prob looks like torch.Size([1457])\n", + "torch.from_numpy(rewards) looks like torch.Size([1457])\n", + "rewards looks like (1223,)\n", + "log_probs looks like (1223,)\n", + "logs prob looks like torch.Size([1223])\n", + "torch.from_numpy(rewards) looks like torch.Size([1223])\n", + "rewards looks like (1310,)\n", + "log_probs looks like (1310,)\n", + "logs prob looks like torch.Size([1310])\n", + "torch.from_numpy(rewards) looks like torch.Size([1310])\n", + "rewards looks like (2446,)\n", + "log_probs looks like (2446,)\n", + "logs prob looks like torch.Size([2446])\n", + "torch.from_numpy(rewards) looks like torch.Size([2446])\n", + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vNb_tuFYhKVK" + }, + "source": [ + "### 訓練結果\n", + "\n", + "訓練過程中，我們持續記下了 `avg_total_reward`，這個數值代表的是：每次更新 policy network 前，我們讓 agent 玩數個回合（episodes），而這些回合的平均 total rewards 為何。\n", + "理論上，若是 agent 一直在進步，則所得到的 `avg_total_reward` 也會持續上升，直至 250 上下。\n", + "若將其畫出來則結果如下：" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 281 + }, + "id": "wZYOI8H10SHN", + "outputId": "80307382-3743-4f70-e08a-66c5e92451da" + }, + "source": [ + "end = time.time()\n", + "plt.plot(avg_total_rewards)\n", + "plt.title(\"Total Rewards\")\n", + "plt.show()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEICAYAAAC3Y/QeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOx9d5gcxZn++3XPzCbtSqucAyAJRAYhkjFgkgAbsLE5MLYB+36cbbizjW0MxhljY5992DjgA4wDDphzgiNLHBkEiCSShFYJZa3S5t0JXb8/uqu7uro6TFjtjqbe59GjnZ7u6uru6be+er9QxBiDhoaGhkZtwRjqDmhoaGho7Hlo8tfQ0NCoQWjy19DQ0KhBaPLX0NDQqEFo8tfQ0NCoQWjy19DQ0KhBaPLXqHkQESOi/Ya6H6WCiE4iog1D3Q+N6oImf41hCyLqFv5ZRNQnfL445JiKEiERPU5E/c45txPR34loUqXa19AYKmjy1xi2YIyN4P8AvAvgA8K2P+7Brlzp9GE/ACMA/GgPntsHIkoN1bk19i5o8teoOhBRHRH9hIg2Of9+4mxrAvAggMnCDGEyES0goueIaDcRbSainxNRptjzMsZ2A/gngMOEvuxPRIuIaCcRrSCiC5zts5zzGc7n24hom3DcnUT0eefvy4jobSLqIqLVRPRvwn4nEdEGIvoKEW0B8BsiaiCi3xLRLiJ6C8BR0v35ChFtdNpbQUSnFHutGns/NPlrVCOuA3AMbBI+FMACAF9jjPUAOBPAJmGGsAlAAcAXAIwFcCyAUwB8ttiTEtEYAB8C0OZ8bgKwCMCfAIwHcCGAXxLRPMbYGgCdAA53Dn8vgG4iOsD5fCKAJ5y/twF4P4AWAJcBuImIjhBOPRHAaAAzAFwO4JsA9nX+nQHgEqGPcwFcCeAoxliz8/3aYq9VY++HJn+NasTFAL7DGNvGGGsH8G0AHw/bmTH2EmNsCWMszxhbC+C/YZNvUtxMRB0AtsMeQP7d2f5+AGsZY79x2n4FwN8AfMT5/gkAJxLRROfzX53Ps2AT/WtO/+5njK1iNp4A8AiAE4TzWwC+yRgbYIz1AbgAwA2MsZ2MsfUAbhb2LQCoAzCPiNKMsbWMsVVFXKtGjUCTv0Y1YjKAdcLndc42JYhoDhHdR0RbiKgTwPdgk3hS/AdjbCSAQwC0ApjqbJ8B4GhH3tlNRLthD0yc7J8AcBJsq/9JAI/DHnROBPAUY8xy+ncmES1xpKPdAM6S+tfOGOuXrn+9dP0AAMZYG4DPA/gWgG1EdBcRhd4bjdqFJn+NasQm2MTLMd3ZBgCqMrW3AFgOYDZjrAXAVwFQsSdljL0O4LsAfkFEBJuAn2CMjRL+jWCMfcY55AnYFvxJzt9PAzgeguRDRHWwZws/AjCBMTYKwANS/+Rr2gxgmvB5utTPPzHG3gP7HjEAPyj2WjX2fmjy16hG/BnA14hoHBGNBfANAH9wvtsKYAwRjRT2b4atv3cT0f4APoPS8TsAEwCcA+A+AHOI6ONElHb+HcV1fcbYSgB9AD4Ge5DodPp3Pjy9PwNbpmkHkCeiMwGcHtOHuwFcS0StRDQVngwFIppLRO9zBpV+5/xWGdersZdCk79GNeK7AJYCWAbgdQAvO9vAGFsOe3BY7UgxkwF8CcBHAXQBuA3AX0o9MWMsC+CnAL7OGOuCTdQXwp55bIFtZdcJhzwBYIejzfPP5PQZThv/AZvQdzn9vDemG9+GLfWsge0fuFP4rg7AjbD9E1tgO6KvLeFSNfZykF7MRUNDQ6P2oC1/DQ0NjRqEJn8NDQ2NGoQmfw0NDY0ahCZ/DQ0NjRpE2UWiiGgagN/DDn9jAG5ljP2UiEbDjqqYCTu9/ALG2C4nPvqnsBNZegFcyhh7OeocY8eOZTNnziy3qxoaGho1hZdeemk7Y2yc6rtKVAjMA/giY+xlImoG8BIRLQJwKYBHGWM3EtE1AK4B8BXYtVdmO/+Ohp2Ac3TUCWbOnImlS5dWoKsaGhoatQMiWhf2XdmyD2NsM7fcnZjltwFMAXAu7IQYOP+f5/x9LoDfO3VMlgAYpeuja2hoaOxZVFTzJ6KZsKsYPg87VX2z89UW2LIQYA8MYl2SDc42ua3LiWgpES1tb2+vZDc1NDQ0ah4VI38iGgG7RsnnnTR2F8zOJCsqm4wxditjbD5jbP64cUrJSkNDQ0OjRFSE/IkoDZv4/8gY+7uzeSuXc5z/+UIWG+EvSjXV2aahoaGhsYdQNvk70Tu/BvA2Y+y/hK/uhbfIxCUA7hG2f4JsHAOgQ5CHNDQ0NDT2ACoR7XM87IU0XieiV51tX4VdXOpuIvoU7CJUFzjfPQA7zLMNdqjnZRXog4aGhoZGESib/BljTyO8Nnpg7VBH/7+i3PNqaGhoaJQOneGroVFFyOYt3L10PSxLV+PVKA+VkH00NDT2EH7+WBtufnQlGtImPnCoXp1Ro3Roy19Do4qwrdNeyrerPz/EPdGodmjy19CoIljO4ktG0SsQa2j4oclfQ6OKwKV+gzT7a5QHTf4aGlUEbvlr7tcoF5r8NTSqCDrKR6NS0OSvoVFF4NyfLVhD2xGNqocmfw2NKgKXfbJ5Tf4a5UGTv4ZGFcHhfgxo8tcoE5r8NTSqCAVH9xnIafLXKA+a/DU0qggD+YLvfw2NUqHJX0OjitCX4+SvLX+N8qDJX0OjitDnyD3a8tcoF5r8NTSqCP1Zx/LXmr9GmdDkr6FRRejN2QXdtOyjUS40+WtoVBH6slr20agMNPlraFQR+rXDV6NC0OSvoVElYIx50T5a89coE5r8NTSqBNmC5SV5adlHo0xUhPyJ6A4i2kZEbwjbRhPRIiJa6fzf6mwnIrqZiNqIaBkRHVGJPmho7O3oF6x9LftolItKWf6/BbBQ2nYNgEcZY7MBPOp8BoAzAcx2/l0O4JYK9UFDY68GY145Z03+GuWiIuTPGHsSwE5p87kAfuf8/TsA5wnbf89sLAEwiogmVaIfGhqloDebR192+MsoAvdr2UejbAym5j+BMbbZ+XsLgAnO31MArBf22+Bs84GILieipUS0tL29fRC7qVHrmPeNh3H09xYPdTdiYYmWv3b4apSJPeLwZfZ8tagliBhjtzLG5jPG5o8bN26QeqahYaOzPz/UXYjEii1duP9125YyDdKyj0bZSA1i21uJaBJjbLMj62xztm8EME3Yb6qzTUNDIwRn/ORJ9+/6lKFlH42yMZiW/70ALnH+vgTAPcL2TzhRP8cA6BDkIQ0NjRikU4Yb8qmhUSoqYvkT0Z8BnARgLBFtAPBNADcCuJuIPgVgHYALnN0fAHAWgDYAvQAuq0QfNDRqBSmDoLlfo1xUhPwZYxeFfHWKYl8G4IpKnFdDo1xUowVtGlSV/dYYXtAZvho1jd7s8Hb0qpAy7NdWjPvX0CgWmvw1ahrVEN8vwzQIQHXOWjSGDzT5a9Q0eqqQ/FOmQ/7a8tcoA5r8NWoaPQPVKPvY5F8q9/dm81izvaeCPdKoRmjy16hp9Fah5W86mn+pss+nfrsUJ//ocffzrp4stnX1V6JrGlUETf4aNY2eqnT42pa/VaLp/9zqHQA8h/Hh1y/CghserUznNBKhoy+HzR19Q9oHTf4aNY3egWq0/B3yL7PCw3B0GC9duxN/efFdAEBnf26IezN4OP2mJ3Ds9/9vSPugyV+jplGNln/aLM/y5xiODuMP/+o5fOVvr2Pp2p044juLfNbxDfe/he/e95Zv/427+/Cjh1cMi7DX1e3dOPOnT2F3bzZ2362dA3ugR9HQ5K9Rs8gVLNy/rPoqi7ihnmUSXrkzh8HEpo5+5C2G7V0ekd721Brc/vQa336Pvr0VP3+sDZs7ht5n8YvHVuHtzZ1Y9NbWoe5KImjy16hZPPjGFjzxTvWVC+dJXuVa/vlhzP75gt23XEwfs0510+FQ5ZTsMbmoKKyhlN40+WvULCzhxXOM6apApTT/Ycz9yHHyjyH1rLNff27ofTf8N1TMoDyUsqMmf42ahVgW2WL+wSAJVmzpwpzrHsSGXb2V7lokyo324ShGNuoeyGPtHswNyBXsvuVjnkkub38/PMjfyb8o4hg54ODB1zfj8t8vrWCvwqHJX6NmwRdEv+z4mQCK19D//MK7yBYsPPzmntV4K1XeoRjZ52O3P4+ThNyAwYYr+xTiLH+bPIeT7GMxhkVvbcX27ninrmz5f+aPL+ORPeQz0OSvUbPoc6zF5jq7uO1wDH1UIW1WRvMvRvZ5df1uAHuumBy3/Pn/cfsNB8sfsNm/d6CA//f7pbjkjhdijwjLMN8T91mTv8Zeh1fe3YVdPfHhdryoW5ND/nESA8eO7gG8uamj9A6WCVfzL4IffvzIChzzPX8iVynRQv17aO1g7ujNx1n+ea75h++3paMfC3/yJJZv6Yxs66E3tqCrjNwCrvnzvq9uj5fJekLyTPaEIaLJX2Ovwwd/+SwuvHVJ7H79+QIyKcO1pAsxVibHWTc/hbNvftqd5qvwztYuXP3X1wblJU6VIPv87P/asKXTHw6Z9HpF9O0hC5uTes7iMwA1uQ+40T7h/Xp1/W4s39KF//jzK6H7rGrvxqf/8BKu/uuyyH5ZFsPtT61WWuxc88879zVqcOW5GrykeH+ugP2//qD7fVJDpBxo8tfYq8Cnyyu2dsXu258toCFtulUy48IKOeQEHdUU/bN/fBl3L92A1e3didosBqZb2G3PJ3ntqfUPOKnn8tHRPHxQGIiw/OtSNs29s7UblsWwur0bP3houc/Bz2eB63ZEO++Xb+nCd+9/G0+tDIYIc2OA9ykqgKAuZQLwqsq2dw34Zi+a/DU0ikQxL01/zrLJv8RCaYRw099zWPrb/PvLG7AzgSQVhUqVdC7mejmx7SltnZ+HO6XDZB1X9omw/MVZQ95i+MJfXsUtj6/CKmFgpoRhmu5go3Awc8ufh59GPR8+IIVp/nFyVyWgyV9jr0IxhNaXK6A+bbgySqnW1t9f3oi/vrQBAPDyu7vwbNt2V4/v6PM05PU7e3HV3a/hyj+9nLhty2IBC9JN8tqDtX3Szjn3VBVUTvZxDt0klr/4XPOWBXJIWpTBogZyf1vcFxG8d3wA4QNS1Dgik7/8LLTlr6FRJIp5aWzyN73QySI1cOZEdL+1uRNf+p/XAAAf+uWz+Ojtz7sW5HOrd2CrQzLcItySoBTBlo5+3LToHezz1Qfw8Tue931nFhnnL1qRolQkE87Ma+7Hs23blW3w2UYxK5+9sbEDd7+4PvH+IgYcss/FJHFlY2QhsQ3A/n20NqYB2BKPZTFs6+qPzc5ljKFgMSH/IDjY8AEkK8wKZl5zv1Keq0vbsg8fTOWZhGpwqTSGjPyJaCERrSCiNiK6Zqj6obF3oRgC73fInxNbseUOojRd/r7f/OhKnH7TkwA8WSCJXPPwm1vw00dXAgCeadvh+67YJC9RZhIJX3X8n154V9kGd4r3FiH7vP9nT+Pqv3kOVMYYTvuvJ/DPVzbGHstlnLxr+Qefzb5ffQCPLt/m218FUXorFBgaMjbxrtvRg8Vvb8V7fvAYOp0ZGgtJ0bpp8Urs+9UHXJ+HKgSVR/vITvEoJzm3/GWH9Z4ovTEk5E9EJoBfADgTwDwAFxHRvKHoi8behaiX5tm27T6dtz9nO3zNEmUfeX/RKhaJlUs/JiUn7e6IFcZMs7hon21dnoNa7HNRso9zzv4yZB+LASu3dePzf3k1dl8u47jlGxTkLvY/UvaRLP+ufvvertvRi21dA8jmLXdb2C350/PrAADbu7OBNjkM53ckz0K6+4PPkv9OeZJXtoYs/wUA2hhjqxljWQB3ATh3iPqisRchitA+evvzOOXHTwAAvvw/r+HFtbvQkPEcvsW+cPK5xPLDqtBE16mYwKiLIv9iLX8x07Tg07+D/oTwc5av+Rcz2AQt/+jzRlr+kubf6ZDxmu09AQduWAQVj87hg4TKUODHytJYp4L8+Qw1VPbZWy1/AFMAiGLgBmebCyK6nIiWEtHS9vbqq7yoMTQIs97FqIrO/hweW2HLBaLDt9hoH3nqv2m3p+WLjl4O3n6SEE2VtchhulU9E3XTJSzATzIWY7GzncdWbMNr63d7mn8J0T78uovJSOYyDyfBOF/DQM7C+p29ysFMLA6XLzBX4lnV3u0+Jz64hPWwPm3f8w6nVr9K9uH3Ur5HqoGc78vvTVD22Xst/1gwxm5ljM1njM0fN27cUHdHYw+iP1fALx9vKykkMsx6Xy8UX3ts+TZ3+l6fNl0ZpVhrqyDtv8ln+Xv9SJuE51btcLX+JO91lOWfLnKwEmchIjEVLBZoQ27xst+8iHN/8Yyr+Rfj8OXgkoZ4riffacd9yzaFHsPJ2JN9/PdaHkBXb+/BCT98DDctfifQlvhcC5ZN/lNGNcBiwEvrdgEQBkWh2fauASxxlrysdxy0u53BQiX78N+efI9UA3lBIv9akn02ApgmfJ7qbNPQwOsbO/DDh+xyBMUmMoUR+PqdHjGv2OIlgNWlzIDl39GXw2f/+FJsVI5snYXtnyswXHTbEry4ZieAymn+SS1pkVj6hCStgsUSJ7Zxv0gplr9L/kJ/P3HHC7jyT69g4271OrZunH+I7CMPWjyiSrWQijgQ27JPDu+dMxYA8OJa+5lwy1u8p5/+w0u48NYlbmAAAOzqtck/J53/w7c8izuX2H4B+R6pSkYELX9Z9tl7yf9FALOJaBYRZQBcCODeIeqLxjADf+GzBavoao1h1vD6nZ7lv0kgnN5s3iU2ThJ3PrcWD7y+Bb97bm1R54oqMQAAr22w6wElIv8I2ScVUs//xbU78S///VzAivRZ/lm/FZw0OooPwqVo/gNO5U1RkpnQUgcAuCskusiVfdw4/mhZhP9mVPXxxevv6s8jV2CYProJY0dk3PO4mr9w3LYue0BZubUbDdzy71U7fJc6MwhAQf6KgVy2/OXf+b2vbkoUElwOhoT8GWN5AFcCeBjA2wDuZoy9ORR90Rh+sEJi0Tv6cgFikxFmMW3Y5RG+qM3v7Ml6tX2cY/m+jc4LHwZZ942z1t7daRf6Klf2CVvJ6+q/LsPza3bi3Z3+EgUisfhkH5bc8ufXVkqGr0r2Gdlgx9qL/ghxlscH0mxIqKd8r/li76pCaaKEwqXEkQ1pZEyP/lzNX2h2v3EjAADLt3R6mj+XfSIeojjAAtHRPmHkf8cza3DRbfH1qcrBkGn+jLEHGGNzGGP7MsZuGKp+aAw/iCTBpYJ8wcKh334EV90dHSYYZvnv6BlwrTdRm9/dmxNCPe0XcLkjC+2I8TnImr9sRX/1rP1x0JQW9/M7W+0w0ySWf1jaPxBe3oGTWdDy9/bzkX8hqPmHeTy507SU2j4q2YeTtCjTiYQqW/7ioGMpZix8RqIaNMUBjpN/S0MKmZRHf57l77U7pbUBgC0T1rmyD3f4hg+afdI96orS/J17Is9sAIRKYpXCsHX4atQuRJLgUsEzq2zHW9zi2GEWWd5ibqy6OJ2eO7HZp/nnChbe2mSX/t3aWZzmL5PxvuNG4Ph9x7qf2514+yThlSqpgENV2G3mNfe7xezksEe/7CNo/owldixyjbtPEU//0BtbXL1bhaxb6MzbxgcR8fy+uH0e6ulsE3MVomYsqpmheA5OxI0Z053xAV6egPgIeXdWbut2yzHs7uUO3/D7JifCdQ/Ea/5ZVd4AAV/56zJc8cfk5UCKQWpQWtXQKAOqEgQPvbEFALBg1ujIY8MKYlkWQ8p52fMWw9gRdfjtZUdhv/Ej3MSvvMWwur3HfRHjyF+2muXPRF6UiIgkftokmn/Bsu+VrMPLx2ZDZJ9/u/MlXHXanPjOwLuvslUL2I5RAPj4MTOUx+byDN0DeV+CHa9mKQ6g4iCVK/hJ8e3NXi1+VZRSFFTRTinD8JN/Pij78GvuzebRavllKnHGIgclyM9Xno1YFnP3cWUfxaBqEGH19m5X5qs0NPlrDDuI/M2tab44i2q6nc1bmPO1B/H9Dx2MWWOb3O2MMbeIV8FiLmkCtlV10JSRAOBL8uILfhw0pSVQuhmwF4rhCDgdA+RPbikBEXGyT75gRUbVmILmf+3fX8ddUv0cmWzCHL4A8F+LgqGRKnAyLmW5xGyhgEvueMENqwS8AcmXfauwpu9fthl1qVddKQ6wrztueUdV3wGv/ymDfLJPv2v5i5FBXhim/GxVbYZBTvLyF5oLt/xNIuzuzWFfx/dQaWjZR2PYwVd/xnknuKWlijbhGaw3LXrHd6z4glrMT/7iQixibZ+3NnciYxo4bt+x2NbV75No1m7vwQd/+ayyn0Bw1mEQuX4G3/XFkH9PTESNW4W0wALEDwRDC0VyitPsRc27oLDKS1mcZiBv+YhfhBgyGSbl/P3ljegeyGOfcfbA/ptn1kbmG0Q9F66tp0zD5/B1LX9FO9lCUB7ztxlN/vJMzP/7jrD8DUJHXw6jnEJ0lYYmf41hB5/swx2+zkuicuhxJ96I+pTPqhJDLwsWc+PjAX8JX1Hzb9vajX3GNWFCSz1yBeZz1skOONkalCNSCFCSfxx/RkX6AN5g9dqG3crvZQejaCUXY7mLx3HLtJT486gILdFxG+d/OGCi7Tz/z4dX4Gf/1xa6X6eUXa2UfUxSOnzFWRnvT66gsPxF53RMiK/svC9Is4vvP/A27nhmTeA4g+ykMh4ZVWlo8tcYdlA5fPmL2KsI5eOWf3NdyheBI5JO3mJuTXrAq8AIwFfYras/j9bGjEsM4nRctqjlaB9ZqjGIUK+QfeIS16L0fsAbrDp61evNRsk+xZC/SHiyRl0MosjfF+0TQ/7jndwAwHOeq7BbJn9FAbiUQW4AAODdl+3dWbyxscPXt2zeCjzrtzd14tlV231thkGWdArSjPS/n1ytPi5vIZu3MFJb/hq1Al+oJyd/XgVRYRXzUg1NdSkfgch1bFKi5U+i5e/F+fflCmjImMjwpR2FFzeg3RZky18mf7XlH+fwjbP8ueYfllQmW/4i+cYloqkcniJKsvwj9Hk5+zYK45o98o9uUy6V4H3mVrrs8OXPrmAxvP9nT7t/8/bk6169vQcfve15X5thCJRuSDjgcflPW/4aNQNLJfsIGZyy5bzDsfxH1KV8A4f40hUs5pIm4Nf8XcvfcbQ2pL0wQDk7VIRMCAELMIT84xy+sbKP098wK16eOYgEW0ySnIpgZQs4CZJa/lGEDvhJMMrhq6qTw58DJ/l0iOwjwo04yluRJB2X+Ca3HbemgoxRDZnYfUqBJn+NYQeRX1zZx60MGdTWuexjGuSf4ouWvwXfNN/n8BVkn76sXcdFTf6y7ON/cVWyT106+IrFav4xso8ZR/7S4CGSahz552Kib0opOBZ2zsaM6ba3paMfC3/yVGQ7osM+ekDx9zFbsNwMXf7bMQ3yO3ylZ8dX7uLHR8ldcVJa0PL3a/6iBKmCtvw1agayQ8z+33uB5PotXPbpzxVCNf8CYy5pApLDV1gcpT9XQEPGkwSy+WCCEIdsfarIP2m9fBFR2b1if8MknE5pkBLvQ6zlH+OALUXzD7PSRzak3ee7YVevch8O0yB3JTTAu44ZYxpx9cK5kedTW/5ynL//mIG85baTzVuRZTDKsvwthqZMdMS9jvbRqBlYKs1fICKZHLnl35+zJM3fH+0jx/lzuHH+XPNPm8ikgpq/bPnLRCqHH8ZZdGGIyu4V+xtG5CqHL7/2OGklq4jwEVFqqKcKIxvSrh6vSoYTYRL5Bm/etx+efwimtTb69s0pNHbePi8NHYj2kQh8e/eA++xzcZZ/hMP36Fmjsb17AGfc9CTW7ehx+uOXNVVBASK05a9RMxDfM0sI9eTvvly8i6fc9+cLoZq/Hecvav4ekYiaPy/fm0Tzl8lRtsSJgCNntOKTx8/CpcfNDL9gCWXLPs7x97y6Ed+6901k85Zr+cZFpviXPBxch29LfVqQ86LbJYKf/AUSlwcOudxytsDcfTjJmwZFWv7v+cFjePnd3U7fou+byuGbMghrbzwbh04bBQBYsbULv356DSyL4VePrwIAZFIG8gUWOxvT0T4aNQNR9hFL33ILSJZ9vIqT/qgMbgHbcpAc7eMdz63i3mwBFoOP/LNR5B9j+RMRUqaBb3xgHqaMaoi7bKHf6hBOub9hhMQT4T5316v47bNrkStYrnUZZ/nnImSfjGlUNNSzpSHli6WPgiz78P1ThuHW3XG/Czh8LTfT2pV9DAPplNdenHTTm1MPyIwx5XPgA5XoVyAAT7Vtx1+W2ol5dSn7fsbJfHGyUKnQ5K8x7CDKPtwizBUsj/wVtVIA26oTyWlnTxZt27qw/9cfwpubOv0ZvsLxhkEg8gYLf7SP1568NKNMavJqU4ZidpEE3YpcBhG8rbAQQ5nIbMs3Wiri4KT60rpdAR9GfdqIDMdkivwMAPjJ4pWBfU2DUJc2XS1d9K2o0FSXUlr+phG0/OU+5oXrHxBmDHURlr8MuSyGdy6mfA789yNKS0Tk80nVpQz05QrIWwz/8b79cMH8qYF2UgYV9dspBpr8NYYcdy5Zh/uXbXY/++v58/8Fy18iR/6y9+cKPgtyR0/WV1bADJF9ANsS5HKJHefvkL9ACjL5B6p6yrV9hL+LI/+8z1/wkSP9pOA6fGMsf45svuDKPknIf2tnP86/5Vl85g/+apL1aTMy2sdXDiImJDRj2msnR1W15PjsSfvizk8tUGr+adNwid37TsrGLVgBh68c5x8nZ6kK2gH2/dykKL3Mr0uelWRMb6CqS5nub2p0UwZnHzI50E4mNXgUrQu7aQw5vv7PNwAAZx9yNgB1kleuwNBcb5N/bzaPbN7CRbctwaSR9a7c0p/3HHMG2U478eUTQz1lLjYNch2tDWnTlQTEwSQu/l6GaPkbxZB/fw7N9WmXGI7fbyzOO3wKLr7dTipKxSR59eUKPss7W7Awut5OkBqIkVd4ljPgOdI56tOmcmF68dj+fB692TxG1EVTSyZlIGUYnuyjGJQuWjAd+45rwr+esA8AYMPO4BrJpkGoS5nu3wWLBZLTcpbl1uMPK+8Qh95cwW1fxG+eWYNfPLYqsD+fDcjnEKVH8bumuhROnDMOa+xo6qQAACAASURBVG88GzOvud/dLg8elYQmf41hB9Hy538XLE+3zRUYdvVmA8XC+p0pNABMaKnHju6sL1Ii5dNf/WScMsi1/MM0/2LJX+VXSIKegQJGNnjkz2Upua0oqUL8rmeggMkjnQEjRtvOOSUFVGhIm24dJQ45YenMnz6J9Tv78No3T488T13KQNokX0SNjGP2GY1zD5viflbNntKmV6ahMWM6yzQqZB9ngBCreoqWfxwYA5rqzECW9+K3t4XuD8iyj9+PIhJ72GA5mJa/ln00hh18JZ2FUM9Gh/xXt3fjmbbtgeP6cwX35RrfUo8dPQO+yJmwqp6AvSi6q/mLso/gkIyTTGT4sojlE0agayDvG7RMIt9gxUkwm7dDOJvrg8QhVu/s6s+5A2esw9diofuoNH9fUpjFsN6xzsPWVeDIpAyfJa06p5wdrZo9iZo/d4zKy2vmCgyZlAGDbFI2DQIRIZ2AWBuFMEw+8xQRZ5lnJINDvF+y5a88XpO/Ri1BWd7B8sj/9qfX4Kq7X1Mc5023x42ow47urM9aF2UYWfNPGRTi8LVf1riIDBVKd/jm0NLgkYFBUl6CW4LajmB6/qun4Noz9wfgSVuiszYnJDnFDWDtXQOhi6rXpU305yzc/tRqt503NwmLrPjq9ERr6LblbwiWf3B/eS0E1QCaNr1on8Y6PjMMDlAZk1y5jBsBdQksf5H8m+qC8fhbYhb8kclbHOTqkpB/EbOTYqHJX2PYgBOKL9pHyPBtSMerlD0DeaQMwrjmDLZ3Z33hmabhWeMyjaQMw923Pm24JMqJRLUOaxyiyP/Hj6zAr58OlvG1r6GAFsHKNAzyWb1iW2nDQGMmhRGO9c+tYDnsNCn5AwisETDeKajG2/zu/W/jlsdX4ZE3t+D8W7z1DUTCV627ICKTMpEyyD2G3+f7/v09gT5zqBa0Ei1/Lp2oau+nTMO9b5z8xVDPMIgDkEqaWbfDn5ksk71I8NlCIdTyD5d9ohPAykFZ5E9EHyGiN4nIIqL50nfXElEbEa0gojOE7QudbW1EdE0559fw46V1u/D759YOdTdKxu4+W09WxfnnLYaGTPjPlfNs90AepkEY01SHnT0DvlIHpuGJJzKRmAa58fW2w9cfGinnFiSBaKjKksXP/q8N19/3lvK4XN7yhS/aso8HsTQ17yffo15ybHKIcf5FKFAAgO998GAcNm0UjhaW0Hx942489OYW337ijG23s9D5lSfvp/R31KUMmCYF4vxbm7wiZnIIp9LyF+L8uewjS0g5Z4bEZ0zc95NE8xcHoBEK2QcAPnT4FPzg/IOdPvj7XCeQd3/O8pG/+J0cscQxnGWfNwB8CMCT4kYimgfgQgAHAlgI4JdEZBKRCeAXAM4EMA/ARc6+GhXA/yxdj5sSLss3HGBZzLeuK8/UlTN8C5a95mnGNEPlE/7ic8t/2ugGWAx49V1vwRODvEShgMPXJLfoV33a0/x7swV86943saa9J/JaVN3yl5BIzrg5y/JN97lG7bYrtMXb5V9zspItbzHDtxj/AwAcMLkF/7zieB8xr27vcRe65xAtf/4s50xsxkxhaU2OTMpA2vB8CHyQrRfILiD7qDR/0743mZThyjKy5W/LPoZn8TuDgJHgPjQICVYjFLIPAMyb3ILZE5oBAI1SQpZcOTQn5DOIzziM5IdttA9j7G0gqJ8COBfAXYyxAQBriKgNwALnuzbG2GrnuLucfdUmkEZR6M0WSsrAHCo8sbIdn/zti+5nvk6vv7aPF8efMu1KjH1WUFJoqjPRPZB3Lf/j9h0LwF8nxxSiZgIOX4FYGjKe5n/va5vQtq0bD76xGVHIpIzgSl4iYccQzc8eXYkT5ozDYdNGoWAxXzVQwyDfQMJ9ABbzrFf+NbcgQ2Ufp86PSNSZlBEpB3GyFO/R6u09gXsoav58Fpc2SElgdSkDKZNgMft5c2tdJPwkDl9O6PUpA3VpEwb5NX9uOKQMw83z4NeRZAhsTEfLPgAwdkSd2w/ZL+BfJ7jg1/yFZxw2CxlM8h+slqcAEIXDDc62sO0BENHlRLSUiJa2t7cPUjf3LvDyBNWC3b1Z3+Ih72ztwhPvtEtVPb1ibfLqSyK45d89UEDaNDBtdKNvMXeAyz7cUg46fDka0qZTTgB419F0UyrBWYDKMScSfpzl/+NF7+C8XzwDxhhyBeZ76W2y9w8k/DO/H3Mn2pbnSXPHAwjKPpxU7XUN/H1RrTkgIi05SjnkcjxiJBBfZcw01PH0Ym2dnOVZxKIUIvdLNWNxyT9tos6p1CkmmPFcCNGPw59lkgmQ3+GrJv+mupT7PEzpdyL+LuQkxCSW/5A6fIloMRG9ofh37qD1CgBj7FbG2HzG2Pxx48YN5qn2GvTl8lVl+cvT86/f8yYuueOFQEE2bqWmTCP0JeHFr3Z0D7jkduXJ+/n2MYkiHb4cXGtOm4ZrqcUnLQUJNGmGrz9WnrcnyD7kj/Mn8qxgTqCHT2/FkmtPwceOngEgKPv4fAhSX8L0Zo6UwvKPuw6+lKIYjSNCrNJZsBhyBQumVMogkezjbLv8vfvgnMMm2+QvSCt8BlSf9iRDz4AozuHbHEr+pkv+8kJD/sqhVqjDN4zkh1TzZ4ydyhg7SPHvnojDNgKYJnye6mwL265RAfRmCz6rebgjbKDaJqzPes3fXse9r20CYFt5YS/JvEn24t4bdvW51uD5R07Fmu+f5X4WZR+ZR1KuDgyBJPwJOgDQooipB+Au+ygiSYbvV//xuuscBYCtTuigaAEbUlEzg8i1gsXEtYkj61HvOMVly79VqAwpW/BxBMPvQ9zspUcYcLjmb2fSBgdGwyC3vVzBJn95VicPGjL5pwRfyL+esA9OmjseadN23HOLn9dbakibvt8BkMzyF2cfoZZ/JuUbyESI93ZbVz+2dHi/bfH84u/6yBmtyuMrjcFq+V4AFxJRHRHNAjAbwAsAXgQwm4hmEVEGtlP43kHqQ82hL1soafGQOGTzVuxiG6UgLBZ8s1ArZSBvueUfUmZ4Ys7YEXWY2mpXzjSltXr5i2lEyD4qwhfJiId6toTUVq9TSCdJMnz/9Py7uOVxrzzAa+ttB7XP8pcyfAFv8JIHHe5wlGvRiM5amUTjpAV+T2RJQ4a43gEf0FKGAZVSJ2bY5gsWsgUroHuHPaOwz4A9GN69dAPO/Km9Khi3/OvSRuAZJ9L8Bctf7N8CIfKpqc50B4mJI+t9x4v3du2OXtzxjBfeK858RePgb585Dl88bY59jcWGZhWBshy+RPRBAD8DMA7A/UT0KmPsDMbYm0R0N2xHbh7AFYyxgnPMlQAeBmACuIMx9mZZV6DhohjL/52tXRjTlMGYEXWx+37lb8vwj1c24q3vnBGIZigHYZb/5g514kxUSn7KIBwydaRj+RuB7wZgv0j8HQvKPvaWjI/8vb87+3PIpMJlJxWB+kI9I15iMVGIz3rEcxtShq9BXtx/Sjpvgxvn73fgtjZm3IxamTTFWcYPP3wIrv7rMt/3cnx8GMRcCE/2UVelNIyg7BM3CMn3UPVb4G2sdqKzeCE3MXGPz/JUA7YMMdonZRJOnDMOHzh0Mo6c0YqTf/Q4AHvAnTyqAT+98DC8d/Y49OYKSPNEshBJ7ZxDJ2N/x0+jAn+uimCaiqHcaJ9/APhHyHc3ALhBsf0BAA+Uc14NNXqzBTBm645xP5rTb3oS45vr8MJ1p8a2u/jtrQBga6llriVdsBjW7+zFzLFNoZb/po5glUQgWIlRhGEQDpw8Eg+8viVQ0thn+ZM6zM8lOMFMFc/V1Z/H6KZMqCWmGhR8Dt8QRzXgL6C2w4l4SpvkRvSYBvnyEkQHsEzI3MHam8u7xwPAqIa0R/7SNWRSBr5w6hwsmDUa7VIxN7ntKNz/uhcR5ck+hvK3aJLnvM9Z9oImcXH3ySx//zb+WxA1fz6DOWG/sfjyGXPx9uZO3LdMHc0lWv6mQfjdJ+2gRXEmzIMNeB2iVuH4OjM4wDSkTdx80eH4c0gmNSBGWIXuUjZ0hm+FsKq9G5+765XYRSkGE3yqH+f05dKQqK0n2T/RPDkGP3pkBU760eNYv7PXV9tcBGNqPTaqEmPKIOzjRPfIMwduJfuSpaT2OfGIBKSKigkjwDjyj7L8+RrEgO2wBnhooueHkEtTuIuFKM7bkDbRly34+jqqMe1p3maQ/D936mwcu++YyNDCqAEMgK8sNy9KlzJIOWCahlduIV+wkCuw2IxbuR0l+Rsy+XshpG6cv2AMXHHyfhjX7J/9is59n+wjjMDiLKUxJP4fUD8fl9gjfhPesx88y1+Tf4Vw1d2v4Z5XN+H1jR1Dcn7GGHodKydO+im2VAFvrxL+hGdX7QAAtHcPRNZ/SSv05ZRhKB2rgP2yzBoXTCYCPJ4XyzvIvKHS/OUqnoYRPg2Pk32irGaf5d/NtXLPyWuQOs6f7ydDRf7N9WnPSRwRjhhF/sXUJ+LVP1MmKcsyGORl3D7dth3/eGVjrOUvt6MqHudbU6BguY7v+pTpSinyICY/u298YB7OPmSSfVxIlFQ6RB6UoSJ/vi3qfqaEAWqwoMm/CGTzFm59cpUyIYaXyh3MpIwoDOQtN+5aNKjX7ejBYd95BGu3exmqu3qzKAa8vUpEEvGfMmP+pCAZqnh+ed1V+buZY0LIn7wXif+tKukM+F9WuXa9HZ6o7q/qJU9a0plLJICfND2ZgiBOVcgX5x88b33aCKwqZhrkWvxhA599bLgVK17DU1efjMe/dJJyv1GNdjnq5voUJrU0KAfMlGD5X/cP26Efp/nLZCmXVwb8pax39WQ9zT8j1vaRBj/p2Yl5FOL77JMEE77nUQQf9R1/MwaR+3U9/2Lwu2fX4nsPLIdB5C4wwcEHBPHH0t41gLRJGNVYplCeAGJct0jSbdu6sbs3h3cdnR0AdhZL/hW0/EUeiLT8UwYgxarbtdvDyT+KuAApzj+EAEWCk+UzI0TCsPtWuuwjYnuPI/uYhnsunnCmaltN/qa7brGIMMtfJCHx9/v0V07GFkFCE6N9po1uBGDfR9km+N1lC/Dc6h04bd4EjGxMK6/dMChogccQapLIF5H8t3dnXcu/LuXJPnGWv2nAjVAynNIRdvlsdSRYHH7/yQX4tztfcvsiLkQj/i+Cv2ta9hkm4KUCVLKJamGNo25YjMOvXzTo/QL89dvFl573VfRF7C6S/PlgUskcAubU7AmDKqM2iRU1e/wIHDp1pO87EvYRpRTV+aKm8GKUjQzVjE/cM04v53Atf8M7l2iJcgQTljxw8pfLJIuObxHiRzHyZ2prI+bP9EIaVbMX1f2aOLIenz5xX+w7boR9XpX/Rojz54jzVSWRnURn/46eAXdG3pAxXeNAPq9sxYv3mwGY7gx0cjXVpHjvnHG46/Jj3M/8Ol3yVxA8f3Sa/IcZVD9RTv6yDLmncq7EWi6Wj/xtSUEk/109nszw0Bv+yowinmnbjmfatrvXUInsYf5TzhasSMtfpe2nTUN57wHvBVp01Ym458r3+L7j749Y1VN+pzg5R03nTQUJu/0tw+HL0ZQxvSgZwdo3KNhf/lkO9QRs2UdVUjklzG5OPWC8sm9R2b7KlbSkbb/62JGY0OKPdVddu2lQoO/cx7Jg5mhcetzMROeXIRphOwTLvyFtYoyT6xDl8wCAKaM8qcpizCV/MXu3WC3+0Gmj8NMLDwPgvYveYBzcn59rsBZvBzT5F4Wox5B1MgrllY5EvLh2p1u8rNIIk324Liouai1q/p/+w0tYs11dsfLi2593140F/L6EUsFfqmzeCo32AdQkXKp+yp+cv7BbiCUd0U6U7BNX2yfJSyzmXMiaf7GWv7j4DC9LYAjt3X7JUbj+vIMCfYuKfVda/tJzmj1hRGAfleZvEAXuNZ+l3v3pY/Gtcw4MHiPsP765Ttkf0UDZ0tnvq9TK768c7cSvIWMa+Ptnj8P8maNd3w5jDNOc5MENu9QhyEnBZx7c6HFlPcX94VLrYGr+mvwjwBjDd+97C23b7LLD7jNSmPMD7kIk4W195FfP+ci0kugNsfx5PXtxgWzRwQgk/4FVUva59Dcv4ran1IuZAGESAwVqp3BEkasX4SM6fNXni5Z9wksCKB2+wqYkJZ3HjvB8QynD8ElUAR9FlOafMl0ivfS4mXjqKyf7+iD7N0RSLTbaR7aiVX4X1aWbRrA9MUNYeX7hJjx59cl4/VtnROwNrNzajb5cARlnIRd+f/ulWRFf0YuB4YjprU6fueUPXHyMXS/ppLnl1RhrFIrrAd4gpJpFWFr2GRq88u4uHPO9R7Fiaxduf3oNLrnjBQDBCBERruwTQk78+7c2dyq/Lxd9OUHzZ6LsE9T85WifpHJOJWWfOKiLeJUWhihq/mGyD29btGRv+pdD8flTZ/vOURT5C38nkQnGFmH5u7KP4p7Upw138ZlpoxvdgANDIn9VLHlktI9ilvHd8/wWurKIW8izlGUf1VKOYe3Up81A4TcR8ya1YMXWTvTnCm6WLb+/chQXf3aiDMmNhILFMGdCM9beeLZbs79UNIYsSxk1gxnMDN+aJf+X1u0KdXz+ZPFKbOnsdxcC6ZXqpMg/UTHbL4wgeXXIwXqWPtnHYrjtydU47DuPqB2+0o8/KalblQj1DLl+mTRUFk/KoFAfShLLX4z2CTp87c+ir+GDh0/F50+dI7RDoQZAbIZvAvJvbVRb/qpBx5V9FIlR9WnTXbhelIVcy5/k/71joy3/4HcLD5qEJ758UuTxKgI7cc64oiJmgOKs4CNntGLl1m70DOTdkhdjHMtf/v3z2ZP42xJlnzD8xymz8auPHZm4T/LAKs/ERFiuUzhx80WjZsn//Fuexdk3Px3YvnxLpxvexjlRFckj4j0/eMz9O4wgeSjoYBVq8ss+wA0PvI3dvTnXyhE1/14peSmpnFOO5b+tqx/rd/aGkqcsXyRJ3fd9F2n5e9NrbyUvddsqS9pwB4/QU8Rq/uLfH3dkBBkj6v11ZMTqkzLxuaGeSsvf9HRlI9gHfq2urmwEBwgVwr7zh4oGrXGZ5Fd97ywcu++YRHWlws4Th4OnjsRA3sKKrd0u6Y5pss8ny55RA3fUb/6q0+Zg4UETE/dJrotlKGZeHKcdOAEAcNbBkxK3Xyxqkvz5A924O+jAWfiTp7Bia5ezn03YnPz5MxK5Uv5xyDXqOTj5l6PhrdjShZnX3I9nV20PfNcX4vBtd0o4iNmQPdkCJgvVB8P6LKMc8l9ww6M44YePhX6fqG6LYYCFxPtE3Vcv2geC7KO2/KMczVHko7JiwzJ8rz/vINxy8RGB/cWyAmmhvINczx/wJAoVcYkWpq8kgZRZyslfNEiiZIaw64+Lf//CqXPwL/O9Su68nSmjGnz7zRjTGHpuIJlvinf/0KmjANhVUrnlzzX/zhDZR8Q4Z2AKq+JaCmTZJ8ry339iC9beeDYOnDwy8F2lUJPkL0og27rUFSQBL1ImivRk6Yi/UPJ0UR5ASgEn/YcV4Zmy7MN/UO3O9YnX3JctYO7EZtzwwYN8fY5D0v0YY3jynXb1lDnk+uWqmmFZoWFdiJoV8G98Dt8wzV9J4uT+H/b8lJcaQv72eYKvXrNg+YtSj+2r8B/PgxDmKKJrxHBNVfau6Vq16r6FIczyFycfqufW2pTBDz58iPLYmy86HPuMbcLiq96Le694j3IfuW1euluFx790Ev74r0dj9vgRGOkQN78fvKy1vMC76pl/+qR98cPzD8F5hykXGiwJsuwTZfnvCdRkhq9IhG9t6sT4ufXK/WQLQfWIdvaonaei86h7IO8udp3kRdvc0YcJzfUBJyEnGNULJtZvt5i9DGBvtuBWifTJPtk8GusaMdmxvKLi7UUktfz/Z+kGXP23ZfjxRw7F+UdOTXSMWFUzWwhJDIog+CQvUMowIlby8g8+/radPgrkX5cyfHKg6s5EhXqqZhg+y9/0wkoNg2CE3PtDp40KbBNJRrxn3PJ0SymzcKeiGHkUdg3uOYpIeJJxzqGTcc6hkxPvf+enFmBuhON1xpgmzHDKfBw1sxWL397mK9r3/06YhfftP8F3jMpPkTYNXHDUtMD2clCM5b8nUKPk771IUQtXd4aEnonSww6Z/J0XShxgLvvNC3hx7S4A8Zp/R18Ox37//3DJsTPw7XMPitxXhLhyU8HyyJ8PGGJ/erMFNAorG8llG97e3OkuhiEiqeXPHeDrVQvAhFnuIZEoIgyKsPwjCIiTm7+wm1pmUkamCI5X/uxl8uf3RlwcXWwpkFWqGMhGSJa/KPsUQn43k0YGreB6sR6NcF+4/GFKz13u25NfPhktDUFqCLvHe5K8TpidPNzyk8fPwkvrduHw6d4Aed3Z8wL7xRWUqxTk84gO/aFAjZK/JfztsYlcB14OCVMZX7Llz18ocR1RTvxAvOyzzVnY43fPrXPJf1tXP97cFB0iKss+tuPN678Y59+bLaAxY7qkJlv+f3tpg/IciatVO+129efR2Z9DS72nm8pTbg4e8+ytsiQ6Ku17H1UuO4nxKabtBzJ8I6bg4jF88KlPm77CYnx72jSQtwqBtpIsROK3/P3RPvLxFy2Y5vPziAiz/Bsky9/V/CXymR6ivcvJUe72ISKvOBy331i88o3TY/cbzKUSVeAzAHHp0KFATZK/aO1zQrnl8VX4wUPLffvJ5M+zd0UOki1/TqQDBfWLGfeiiINJX7aAhoyJi297Hiu3deO6sw4IPc5X3oGxwApCfss/j8a6VMAC5AgboOJkn5yzFB8//NdPr8Gvn16DtTee7e4TFjklxzyLs6vffXIBHlvejmmtjb7tk0bWu7X7oy1/5xwRcf485lz1eMT4eObcZlm/5f6NtEngP5viNX9hnV0p2kfWqb7/IbWGLvdNtOoDso8bSx7alA9hmn+SMNbhjLhqopXEXZcf4xbF4yhHNisHNenwFS3dbMHCW5s6A8QPBMmfzxJEIt3ZHWL5h0TQxGnTIvmvaredeut22PIJn5momgha/v5HmxX6niswNAoLk6za3uOrKR8W8REl+9y3bBNmX/cg3t3RG3mNYVmccj19caCZO7EZ3/jAPBgG4cKjpgMAHvnCe7H4qhOF40NP6Uk9BvmctyJShno74A0IYqatfH95d8XFysln+fvbVMk+8qpR4iI0xfCr6PAVE6lk2ef4/cYCAM48KFk4oW8wUmyvVuwp2QcAjtlnjBvlxKPsBrNmfxRqkvz9so+Fi29fotwvYPk7x4lhkzt7/KthuZq/YOGKzzbuQYszCW4l82lpnyRL5QoWlm+x5aDenGT5S/HW/Jr5INGQ8cj/6/98A8d871F337AeRln+P3xoBQDbWR01vslOdI6PObHvpzvxzeLYKYYrnnf4FKy98WzMmdCMpjpRI4+w/OGRaJjD192usvyFMgsc8szqhNmcSNVx3/KgorI2xQElLSx8rpJ9ohBm+fP1aHlbB0yywwnFxcijICeJcQzmIuN7AkO1Bocnuw3J6WuT/LNSnZtdvTlMaAkmnHQIySDZvFeFUoyc2dGTxaSR9ThgUgsAjyDFAUakzLAXZXV7Nz712xd9uQcDef8CMTL533D/21j4k6ewYVevL9qnYAV/0HmX/O39mgTZB1CntsuISgZ7d6fn3I0a31QLcADAwVNGYu2NZ2OWs+aAGCaaZOGMKAJSyT5hZKpKQvNkH2+bPLgePNXu/1EJiTStkJnEomqmSe5MIyrEVIU42Ue1AlYShC0tOFSWa6WwpzV/DrfGzxANnmVdNRH9JxEtJ6JlRPQPIholfHctEbUR0QoiOkPYvtDZ1kZE15Rz/lIhEjOPTDn1gAmB/USi6s3m3ePEl6e9awDTWhvdpB0+mouOTZEzw96T7z2wHI8u34Z7XtnobstKlj+vUMgJ6qmV7QDsUNLebMF1GBYslebPnOuwB5DGTPh6tGG/xbDFXMRch7hs6DBwAuH6pzjLSFIGIEltn6g4/6gqiqLswyGXPvZmB7FdBaDOKK6XLH+ximjJlr/o8HW2l/qM3Htc3VwfwJ6UfUTwlecuO37WkJy/3KteBOAgxtghAN4BcC0AENE8ABcCOBDAQgC/JCKTiEwAvwBwJoB5AC5y9t2jEPX49TttS5tbnGHoHsi7Gl2uYOHav7+OJ99pR3vXAMa11LkvhrePmijDrOpRjbazb5OwclJ71wC6+nOe7ONY7bwJPhjk8gx9AvnLsk9TxnQHI+4YjlqMvFjZR5yRZPNWZPhsGLyqmn5nJJBs4Yzo2j5i1IyzTdrHy6FQtC3E2/O2ZMvf9RkkZEYvmc3bVieRtrh+QHGWf0iop2P5l/J8xLbOODB5SYNqwFBZ/q1NGay98Wycd3jlEsmKQVnRPoyxR4SPSwB82Pn7XAB3McYGAKwhojYAC5zv2hhjqwGAiO5y9n2rnH4kQa5gr3G7tbMfdy9d727nseh81aEw9AwUkHOifVZv78GyDR14+M0tyOYtvHdEnWu5Woo4fxFhJNXa6EV6NNen0NWfx5f/ugxf++cb7mISsuzDP7uWf30K6Aw6fEePyLj94XXem+pSoVEaoZZ/TN0iwJ7xlGJZyssS+mZLCczpJE5HMVNWtqT56VQWtjt4iJq/RBieJBLbDQBCSKtI/r74fHKcvk4fijC3xaghUyH7hIXbxiGTMvDcte9za+TsLaj2aKVSUclQz08C+Ivz9xTYgwHHBmcbAKyXth+taoyILgdwOQBMnz697M6dftOTAGyHo+hU5Vr1TMHyv+q0OXht/W48unybu60vV3Ct+mUbOuxjxjTi5Xd3Y3xLXSBlPhcW0hjyQxsp1BCZNLIeXf12pM9A3lLIPnA+2+Tfm82jN5vHRKdeT4ExH3mObqrzHL7CsnbF6N7itckQyT6bD5J/ksxgVwaRio4lReKSziRtdGBFZLtyAjbIO0wm/7AoojDw6xTvtUhClsT7JwAAIABJREFURCTJPomaBQBMElbSSiuifUqVfQB1Ulm1YzDLJg9nxM53iGgxEb2h+HeusM91APIA/lipjjHGbmWMzWeMzR83rrxFFABgzfYerNneE4jLZ8wm3tFNXjr7afMm+D4Dts4vr9LFp9Hjm+tdgigoNH8RSWrDzJ3Y4vvOk338lj8n/55sAX25glsbxrKYS2b1aQONadNNOusTNP+w+OLQOP8kln/ech3Vqu/DIC8uXuzCMZHWG3f4Cpp/wPJPLPvY21TVK8OOV4GT8jH7eA7iwOpiVFr9F3GmJA6K5co+cfjgEMkXGqUh1vJnjJ0a9T0RXQrg/QBOYV6IxkYAYmGMqc42RGwfNETV5AaACS11vhjrupQRqMORdeLjRWzabevz9pJy9sscF+cfBnGwmD+jFf/72ib3M9eH+QLynAe4Qd3Rm0WuwHwOX07+LfVppExyJSJX9smkii4yF+bwFcl+IF/AQM5PLnLmtAqcpDJOffqwc4UhShpyHb5iVU9pH/4bUbUSFeq5z7gm3P6J+YF941CfNvHg507AjDGNmPeNh5X7GEShsfVxaKlPobM/L8k+9u9jMMhfTOTTqA6UG+2zEMDVAM5hjImFXO4FcCER1RHRLACzAbwA4EUAs4loFhFlYDuF7y2nD0kQFl7I0VSX8k2P69NmYC3TXIEFQuR4WOa45rpAqYQwzT9usRcAmD+z1fcdt/y7B7zQU1/EkZNoJjp8+Xma61PImIbbnz5B9gmz/MMKvYX1XST7AYXsI/sqVPCWFnQG0SKrR0dZ/qpFUWQy5faBsryDIr6dR9TUpUzsI/iLiuHoAya1BGq8y+c1ipSTOHgEiWjEcNmnVM1fY+9CuZr/zwHUAVjk/DiXMMY+zRh7k4juhu3IzQO4gjE7MZ6IrgTwMAATwB2MsTfL7EMknlrZjvHN6qqdHPXSFL4+bfrC7gBbw5dJkVtQ45rrXNnHiiF/7jc46T8fw0ULpuPfTtzX1xYAzJGqFnJ9ma/OxBiwS8hB2OFk5za5lr9Hns31abveTIFh5jX3u8c0Zkx0D6gHxTB/RegSlcK12g5fP9mHnUeEWNXTvoYiLf8Ecf7ifvL+3pqpqrad/4U8Af5M5EXoK5mpL0b7FIvPnzobFx89HeMF/Z/3WX4+ewqjGtOBd204QZ7t7+0oN9pnv4jvbgBwg2L7AwAeKOe8SdG2rRufuOOFUGssbRJyBRZYC7QuZSgsfyt00RPxR+Nq/iEEygeFtTt68f0Hl7vkz7ff9+/vCcQde5a/TaJ5i/nW4eV/8+soME/2OXjKSOzuy/msPSJ7wOvPhQxQIcQbKvvkoh2+XdLM6wfnH4xcgeFr/3zD3ebKPia3/Cun+XPKZiw8k9eKEP1VMfy8n/K9KiYqh+O1b56urHZqV/Ysujm7H0Q+4ge835FYdHBP4qWvnTYk502CJ798Mprqaov89+oM3/3Gj8B3zjnQterklYP4lFtO2KlLGYraOEGHL0dGqMDYmy3gij+9jDc2dij3LVhM6YPI5i1MbKnHQVOCK/dwouF+hILFfGUSdvXYf/NpPXf4zp/RiuvPOwhpk3zrEDekTV8kSaAvYZJV6OL0/jh/WfOXLf+0aQTKG3iWf2nkH6n5O18xFr6S1wGT7NnWvEnBWvF+2Yj7JoLJaOK5VPjUe2bh+vOCZbpHNqQxsjG4YlTUMyoFfFWq9zilKPY0xDLVww3TxzQWvaxktWOvr+p5wVHT8PV7bGVpv/EjfOUTGjMmOvpygaloygxa/vkCUzpxifwRFYvf2oq3NoeXX84VLCW55gosNNlEJqq8xXxrDbiWv9PngmVr/py0MqaBzj6PgN3qjiGZs2Hp/2GWvzjLGVBE+3RLlv/U1kZs3O2v9S9X9SxWlk4Sq83AvJBM6buFB03C4qvei/3GK8jfDfX07Ho+O5Nng1EyzdffX1w+oxjtUwmMqEvhqatPxnhFKRON2sNeT/5iSJ5cv4eToEz09nHBksgqyz9jGiDypuec+BvSptLRmbeYMs46m7dCyxgE1wm2XDLPmIa7IDWfwRQYg2V5pJU2DV9f3LruIUQVFqkk9mN3b9b2jaTN2Dh/0VENAEfOaA0sn5lyl1H0LP+ZYxqxpTN8mU0RSS3KqAJuKuIHoCyxkA7JR6hkyPi5h09RrtRVDuRywhq1i72e/AG7hvaLa3ZiW5e/AieXfRoU5C/Xaw/T/Lm1zotvcS5ozESQv0Jrt5O51Jqj7D8oCJb/pFH12OoQJO/zsg270ZPNuyUj5OUPm5zrDiPMcNnH+/ui257HsfuMwTc+MM/tX33aiNX8F8waDdOgQKSRN1B5pProF09S9kOFKPLng524Em4xjlRxJTCvv/7oLnd7Bdn/5LnjgbkVa05Dw4eaIP9j9hmDY/YZg2//rz+wiFvKsuYPBC3/bIEpI3jE/Uwi5B32DwtvzIfKPlao7CO3lbeYW3F0Qku9W++fk/8flrwLwCszLOvr8opOqj6qIMo+2zr78c7WLgBexmhzfdqO9smpo32+/6GD8WFnTV8ez88h1/O3hEXokyBKG//5R4/AXS+8iwMmNXuhk4lbVtfz5xnZnz1pX9++5VL/3f92bKCUuIbGYKAmyJ/jogXT8Ztn1rqfORFw0vzxRw5Fm7OAitLyF8ivKWOiJ1vwEavB1xtEOPlbLJipC9jWfSZE9pHb4pZ/fdrw1QSSZzByvRwOLneFcWuo7CNIHNm8hU0dtv+Ea/zN9Sm17ONY/iMb0r7FtEXIoZ7FxvlHaeNTRjXgi6fbJrQn+xQxsPAsW4PA6T1jGsrEpnJLBSStra+hUS726mgfGXMmNOP1b3lreoolEADg/COn4isL9weg0PzzftlnVKNd/iEtFePikINVvnjaHHz5DJuAxMgbt/0oy18aLHKO5t9Sn3YlHPs6ZPK3/w+Sv31MGFElSVAbyFvY0tEPxpjrr6hPmXaGb97CJ4+fhaVfs5PDueUv3h9Z9jGlgarYOP+k4MJPMRytKuwWhmEazKKhEUBNkT/gJ0huycdp/kR8+UOPFFubbItbtPyjyCGdMlwLUgx9XLJ6BwBbZw+rKx5m+bc0pH3O6oaM2pqW45fjklnCyJ/LPowxZAsWerMFdPblbX+FaSCTMrD47W3oHsijLm2495Br/uJAKZOkK/sYnsN3MOBa/kUINKriamG9q9UiYRrVh5ojf5FgOZnFRfukTQM5i/lkn1bH8hetdVl6EHkgbRqu5ds74JH5hbcuwdrtPY7sk8zy56GeLfUpXz/lYmNc9mmp98eQh2VZ/uCh5ZHJbFz2EWWdb9z7Bn799BrUpU3fvegdyLt94zWJxLr8wXslyz6DS/7FWOjuMUJ5iLDuactfo1pQc+QvgpO5LJcA8nqqhFzeb/mPUpC/7KAUZxQZk9yBp0eSfTZ39CNbhMPXTvLKo6Uh7eu7nKnskn+D37UTZpze8vgqPPD65ljLX3RYP/LmVgB2lI54z845bLJ7vU++Y684JkYdhXFkseUdzjtscqL9vPMWL/twiHH+oe1r8teoEtQ0+XOCkev4AP4BIZ0yXIvYi/awCVW01mVrVbS406Yn+/QM+Ml8d2+2SMvfciz/tC9SSR7E+Plkyz8K3QN5pcPXNEhZuoIPTGIfv3jaHBw5w3ZcniuQszjrCpNH+OzghNnJynjf9C+HYfX3zkq0L+CFlBYjz/DH6h/ci1upTUNjuEGTP4IWMxCUfbIFhrxl4dMn7ou1N57tSieitV6QSDMjtcFj2HukcgdbO/uLC/Us2OUdWhpSfstfdvhy8m/wk3+UotKfs5SWv0nkZt2qktQ6+/Nu9VQxg1QsZyAmsYVxpGEQHv/SSfjVx44M76QAvuhJUpRt+cfKPpr8NaoDNRXqKaPAwmWfep9kYycv5QrM1e15DRrRWpdr36QMQn3aQH/Osp2dDufLtW62dA44ETPJxuJcwUJHXw6jGjK+WUsw1NP+ny/ykgT9uYKysJthCIvThxSt4wu5i1VUm+u8c4vXF0WSM2PWUy4HpTh8OZKMMZr6NaoFNU3+liv7xFn+hL6c47R0Sd9f4AsIZnumTEJrYwabO/qRMQmMOQ7frMryD6/tI2NXbw4Wc0rkioOUvK5siMOXhcaq2OSvInfb8o8jfzs5aVyzZ/mLMoho+R88ZSQuPno6Zo1tchfF2RMgReROUiRJOtOWv0a1oKbJ33P4Bkk3ZfolG05QY53Kf3wQEAlBdlKahoGRDWls7uhH2jRc2aRH0vC3dPTbmr9A3j/88CH4yaJ3sKkjSIy8fv+oxowvdl4mJ050suUfJfv0ZQvKGkaG4ZE/T+qS6xfxzNSwwmGi5W8ahBs+eHB4RwYJ/A6V7vC1DwwP9SypWxoaexw1qfn/z6ePxV8uP8az/BWyj4i0abjVQHlxOD44iJatTP5px/IHbPJMRWj+cpz/BfOn4W+fPU7ZHz54tDamA8loInhz4kB2wuyx+Nyps0OP6c8XQh2+suwzY4y/SFiT4zsZ06Qm/1RCWWswYZQg+/CZkkHx5K7JX6NaUJOW/1Ez7UgUrtHLhc9kpFMG2p2icBOcBTK4hBF1pGmQmwzW1Z/HCCfZSiZ/3rZM5GGLhHOMakz7SjWrzi/jzk8dHdlmX9Z2+F5y7Axc+b7ZOOqGxXZbJFr+NvnPGtuE5Vu63GP/ecXxeHX97lB5JD0MguArJftoh69GtaMmyZ9jzoRmbO0cUGb4ihBr7ox39Gyvmmf4cSmD3BK6BcuCadgDgRzq6SZBSYOQOBiIFUM5RjVmlJE33jHFE1F/roBcwUJd2nSLlwH2zEW2/E/efzw6+nJYeNBEzBzThNkTmjF7QrAs8oi6FLoH8kUVahssuD0o4d4QEY7ZZwwefGMLZo5Vl0bW3K9RLahp8v/5R4/Amxs73IStMPA6NAbBXe2Hb4t611OGgS+cOgejGzP4wCGT8cKanQCCSV4ccpy/6ANorksFFqIf1ZCOrABZyipQ/bkC8gWGtEl+f4LC8j9o8khcMH9abJtfPmMuvnnvm+4aw0OJUjJ8OUyDcPHR03HqvAmBVeE4tOWvUS0Y+rdxCDGyIY3j9gtf0u6U/cdjn3FNWLHVrvQ5rrlOKD3M5YPwlz1lEurTprtOL9e8wxY0l+v5i+TdXJ8OkP/IhugFsUWCu/S4mYG1dFXoHsgjbzGkDMONnz9p7jis3NrtOsi5wzdpdNIlx83EJcfNTLTvnkIpoZ4m2cs4hhG/3a6GRnWgLA8cEV1PRMuI6FUieoSIJjvbiYhuJqI25/sjhGMuIaKVzr9Lyr2AwcSvLz0K1509z5V9JggLYhtewHgo5KUFeUG1XT1Z1e4BMhVrycgRO831KaRMQxmpJB7P8a1zDsSPLzg0vLMO+EyC9+XZa96HX33sSDvOXwr1jHI2D3cUY6BHrO2uaFfTv0Z1oNy39z8ZY4cwxg4DcB+AbzjbzwQw2/l3OYBbAICIRgP4JoCjASwA8E0iai2zD4MOHoUjkn9U5RlOnKZUtphr6Lt6PammScgunjyyHjL4ACKTP48iiopUKkX24Q7cMU12+5NHNaA+bdqyDwN29mTd2j7VTP6DF+dfQmc0NIYAZb29jDFxpfImeJx4LoDfMxtLAIwiokkAzgCwiDG2kzG2C8AiAAvL6cOeAJdrprZ6033GePhf8G3nFr7swJXLLACeDwEApo8JOhFNl/z9x452yDmKgEtd/LspY+KsQyYF2lq5tQtHXL8Iv392nXPuaEf5cEYpFnoS8teWv0a1oGzNn4huAPAJAB0ATnY2TwGwXthtg7MtbLuq3cthzxowffr0crtZFnodjX5qq0fOfJRTveoNaRO7kQuQRXNdKhC1M2ZEBu/utJdhnDQyqCXbjmUrUH+I685Rln8pzsf9xo/AhUdNC2QFm0RYvb0HALDCWb4xqeY/nFCMhCMjCbEPpuU/Y0wjZocsMq+hUSxiyZ+IFgOYqPjqOsbYPYyx6wBcR0TXArgStqxTNhhjtwK4FQDmz58/OMXdE2Jrl51l63P0RZAIDx2Va/UYBgWidkYLkUYqy5JvM4nwm0uPwv8u24S/v7wRk0fZElGk7FMkN2dMA4uvOjGkLQqUdahG8uco1eE7GO0mxRNfPjl+Jw2NhIh9exljpzLGDlL8u0fa9Y8Aznf+3ghAjAGc6mwL2z6ssa3TTsISZR8e86562aMWSB/prLnLHbVxBMo1f4PsuHq+bCOfJURJEUVb/hG7Rw1M1QRvDd/kx3DLI1Fht+q7JRo1inKjfcQ6AecCWO78fS+ATzhRP8cA6GCMbQbwMIDTiajVcfSe7mwb1tjmZOBOaw1q8qqXPUzzB4DmOpv8eQkEwyBccuwM/PDDhyjP7S4e7pxoa6c9C5mocA7LKJb8o8itGoleBS77lHI5SXwomvw1qgXlav43EtFcABaAdQA+7Wx/AMBZANoA9AK4DAAYYzuJ6HoALzr7fYcxtrPMPgw6rjx5P/z8sTbfilhzJtra64JZowP7NzjWuYowuRU5ZkQGG3f3IWUQvn3uQYH9OFzyd/7f5paZUNfPEVEs+UdJFt6SkMFks2rEYMk+OslLo1pQFvkzxs4P2c4AXBHy3R0A7ijnvHsaXzpjLr50xlzftiOmt2LJtacoLXBeYz9lBCdWPEHq48fMwJLVO/GVhXMD+4gwBdkHsIu5AUHn8BHTRwEAjpzRipfW7XKOjWw6gCSW/6HTRuGplduLa3gYoqSqngnupyZ/jWpBTWf4losw6YU7euUkLwAYyNlO02mjG/GRBKUReBucfH98wWF4ce1OTBacz8uvX+h+/7fPHIdLf/MCHl/RXnSoZ1Q0C7d6J7bEy03VgKJCMl2pSMs+GnsPNPkPAjjpqkoY87o445vjZRvAI31OVqObMjjjQH/wlRzxw/mneNknHLxu/4wxjbj9E/PdEtfVilI4OlmcfwkNa2gMATT5DwLcpR4VZDFvcguefKcdYxOSv1hUrlgUm+Ebtfuqdru+0dyJLTh13oTiOzPMUJLDd4hDPTU0KglN/oMAzhEqS/FnFx2O5Zs7A0lUYRDj/JOCk1QlZZ9eZwGZ/SfuHUlGpWTiJiH/vSQoSqMGUL1ZOsMQ/7ziePz0wsNc608V6jmyIY2j9xmTuE2+0EwxZFVM2eLj9vX6kmT/qIqW1YTSMnzj99EOX41qgSb/CuKwaaNw7mFT3GX/5MJupcCUHL5JwAeKJMfc+amj8bOLDvcdp8Lp8yZgamtDyfWChhuKGUz580xyhOZ+jWqBln0GESrLv1hwuacYzuW7JiE40yC3OFzU3rd+Yn7yDlQBSnkySe6nLuymUS3Qlv9gwAkNrERWrJzhmwSuzyHhMZ60VFzfqhmDJc/U0j3UqG5o8h8E8CxeVahnseDEXIzcwn0OSU/vEWHtMFcpi7kkgdb8NaoFmvwHAbzWvyrUs1iYJYR6esXLkh0kZxHXAgbrUmvpHmpUNzT5DyIqQf6pMkI9kx7D96slo3XQZJ8amj1pVDc0+Q8CPNmnfCIwXGIu3uObNNjIKMGvUPUYpFDPWrqFGtUNTf6DiEpYgXzyUJTD1z22ONmnFnjLq82vHb4atQ1N/oOAYhyEcfCyhYs5pjgGKml2UeUo5kqLeZw1NXvSqGpo8h9EVIIH+OyhqAxf5/+kg5BXPK6YnlUnvByIwWlfk79GtUCT/yCgkgsOc92+uAxf3o9kPdEO38qhhm6hRpVDk/8ggFVQ9+EWfzGBQ5zYknaDDzC1ZLUWc6kHTm4BALQ2ZirarobGUEKXdxgEcM6thIZerPNWPMYqVvZJ3q2qRSnP5rqzD8C5h03BfuNHxO5bS34TjeqGtvyHOdzyzEWVd+CWf3GyT01Z/kXsW5cyceSM1kHri4bGUECT/yCiElRaTHlm+Zik4lMtVXcYbIevhka1oCLkT0RfJCJGRGOdz0RENxNRGxEtI6IjhH0vIaKVzr9LKnH+YYcKenzdbN1SsoUT9oNPEGqBDwc7zl9Do1pQtuZPRNMAnA7gXWHzmQBmO/+OBnALgKOJaDSAbwKYD/s9fImI7mWM7Sq3H8MRlQn15G0VU97B/j9ptI9VxALlewtq50o1NNSohOV/E4Cr4bczzwXwe2ZjCYBRRDQJwBkAFjHGdjqEvwjAwgr0YVjh6H1GAwBmjmkqv7GSMnztfZM6fN3FSmqIEWvpWjU0VCjL8ieicwFsZIy9JlmmUwCsFz5vcLaFbVe1fTmAywFg+vTp5XRzj+Pjx8zAKQdMqMiSh57sk/wYV/MvUvapKcu/hq5VQ0OFWPInosUAJiq+ug7AV2FLPhUHY+xWALcCwPz58yuZNzXoIKKKrXVbzopTVkL2n9BSDwA473DlOLxXQlO/Rq0jlvwZY6eqthPRwQBmAeBW/1QALxPRAgAbAUwTdp/qbNsI4CRp++Ml9LtmUGzCFlC8pDG6KYPl1y90l3OsBQz2LGfa6L1joXuNvRclv+2MsdcZY+MZYzMZYzNhSzhHMMa2ALgXwCecqJ9jAHQwxjYDeBjA6UTUSkStsGcND5d/GXsvePZtUv0eAM45dDIA4Jh9xiQ+pj5t1pQUMpiX+uDnTsC9V7xn8E6goVEBDFaG7wMAzgLQBqAXwGUAwBjbSUTXA3jR2e87jLGdg9SHvQTFSTiATfprbzx7sDq0V6CSlVdlHDCpZfAa19CoECpG/o71z/9mAK4I2e8OAHdU6rx7O7ywTQ0NDY3KoXZE3iqFK08Mpqlag6ghhUtDQwlN/sMcPGZfU7+GhkYlocl/mIPLPlYxHl8NDQ2NGGjyH+bwYvaHuCMaGhp7FTT5D3MUW6FTQ0NDIwk0+Q9zuJq/dvhqaGhUEJr8hzlczV+Tf0WgB1ENDRua/Ic5ii3SpqGhoZEEmvyHOdwlGYe4H3sLaqmEhYZGFDT5D3Noy7+y0LKPhoYNTf7DHHPGNwMAZo5pHOKeaGho7E0YrMJuGhXCh46Ygn3Hj8Bh00YNdVc0NDT2ImjLf5iDiDTxDwK09K9R69Dkr1GT0NK/Rq1Dk7+GhoZGDUKTv4aGhkYNQpO/Rk1Bx/lraNjQ5K9RU9Bx/hoaNjT5a2hoaNQgNPlraGho1CDKIn8i+hYRbSSiV51/ZwnfXUtEbUS0gojOELYvdLa1EdE15ZxfQ6NUaOlfo9ZRiQzfmxhjPxI3ENE8ABcCOBDAZACLiWiO8/UvAJwGYAOAF4noXsbYWxXoh4ZGYmjpX6PWMVjlHc4FcBdjbADAGiJqA7DA+a6NMbYaAIjoLmdfTf4aGhoaexCV0PyvJKJlRHQHEbU626YAWC/s8//bO7sQK8owjv/+brqFRqUtIa3UKgshEZtsYfRB2JdZtAleCEFdBEUpFBGlCFEXXRT0CVFomfatWZEIUZZCV7mu5ceaaVZGiblWaHXTl08X82xO2zlHlz0zczzz/GA477wznPnt/8x5nHlnjvO991Xr/x+SbpPUJ6nvwIEDddAMgiPEsE9Qdo5a/CV9KKm/wtQDPAtMAbqAfcBj9RIzs8Vm1m1m3W1tbfV626DktJ7QAsCoqP5ByTnqsI+ZXXksbyRpCbDGZ/cCk1KL272PGv1BkDkPzz6XyW1jubQzDiiCcjPSu30mpmZnA/3eXg3MldQqqQPoBHqBjUCnpA5JY0guCq8eiUMQDIcJ41q5b+Y5tIyKI/+g3Iz0gu+jkrpInjK4B7gdwMy2S1pJciH3L2Cemf0NIGk+8D7QAiw1s+0jdAiCIAiGiY6Hn7t3d3dbX19f0RpBEATHFZI2mVl3pWXxC98gCIISEsU/CIKghETxD4IgKCFR/IMgCEpIFP8gCIISEsU/CIKghBwXt3pKOgB8O4K3OB34sU469SS8hkd4DY9G9YLGdWs2r7PMrOLP2Y+L4j9SJPVVu9e1SMJreITX8GhUL2hctzJ5xbBPEARBCYniHwRBUELKUvwXFy1QhfAaHuE1PBrVCxrXrTRepRjzD4IgCP5LWY78gyAIghRR/IMgCEpIUxd/STMl7ZS0W9KCgl32SNomabOkPu8bL2mtpC/99bSjvU+dXJZKGpDUn+qr6KKEpz3DrZKm5ez1oKS9nttmSbNSyxa6105J12ToNUnSekmfS9ou6S7vLzSzGl6FZibpREm9kra410Pe3yFpg29/hT/QCX/o0wrv3yDp7Jy9lkn6JpVXl/fntu/79lokfSZpjc9nm5eZNeVE8rCYr4DJwBhgCzC1QJ89wOlD+h4FFnh7AfBITi6XAdOA/qO5ALOA9wAB04ENOXs9CNxbYd2p/pm2Ah3+Wbdk5DURmObtk4Fdvv1CM6vhVWhm/neP8/ZoYIPnsBKY6/3PAXd4+07gOW/PBVZklFc1r2XAnArr57bv+/buAV4D1vh8pnk185H/hcBuM/vazP4A3gB6CnYaSg+w3NvLgRvz2KiZfQz8fIwuPcBLlvAJcKr++/jOrL2q0QO8YWa/m9k3wG6SzzwLr31m9qm3fwV2AGdScGY1vKqRS2b+d//ms6N9MmAGsMr7h+Y1mOMq4ApJdX/OZg2vauS270tqB64Dnvd5kXFezVz8zwS+S81/T+0vRtYY8IGkTZJu874zzGyft38AzihGraZLI+Q430+7l6aGxgrx8lPs80mOGhsmsyFeUHBmPoSxGRgA1pKcZRw0s78qbPtfL19+CJiQh5eZDeb1sOf1hKTWoV4VnOvNk8B9wGGfn0DGeTVz8W80LjGzacC1wDxJl6UXWnIO1xD33TaSC/AsMAXoAvYBjxUlImkc8BZwt5n9kl5WZGYVvArPzMz+NrMuoJ3k7OKcvB0qMdRL0rnAQhK/C4DxwP15Okm6Hhgws015breZi/9eYFJqvt37CsHM9vrrAPAOyRdi/+BppL8OFOVXw6XQHM1sv39hDwNLODJMkauXpNEkBfZVM3vbuwvPrJJXo2TmLgeB9cBFJMMmJ1TY9r9evvwU4KecvGb68JmZ2e87eR+UAAABeElEQVTAi+Sf18XADZL2kAxPzwCeIuO8mrn4bwQ6/Yr5GJILI6uLEJE0VtLJg23gaqDffW7x1W4B3i3Cz6nmshq42e98mA4cSg11ZM6QMdbZJLkNes31Ox86gE6gNyMHAS8AO8zs8dSiQjOr5lV0ZpLaJJ3q7ZOAq0iuR6wH5vhqQ/MazHEOsM7PpPLw+iL1D7hIxtXTeWX+OZrZQjNrN7OzSerUOjO7iazzqufV6kabSK7W7yIZb1xUoMdkkrsstgDbB11Ixuk+Ar4EPgTG5+TzOslwwJ8kY4m3VnMhudPhGc9wG9Cds9fLvt2tvtNPTK2/yL12Atdm6HUJyZDOVmCzT7OKzqyGV6GZAecBn/n2+4EHUt+DXpILzW8Crd5/os/v9uWTc/Za53n1A69w5I6g3Pb9lOPlHLnbJ9O84r93CIIgKCHNPOwTBEEQVCGKfxAEQQmJ4h8EQVBCovgHQRCUkCj+QRAEJSSKfxAEQQmJ4h8EQVBC/gFtzJkqOWV8fAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mV5jj4dThz0Y" + }, + "source": [ + "另外，`avg_final_reward` 代表的是多個回合的平均 final rewards，而 final reward 即是 agent 在單一回合中拿到的最後一個 reward。\n", + "如果同學們還記得環境給予登月小艇 reward 的方式，便會知道，不論**回合的最後**小艇是不幸墜毀、飛出畫面、或是靜止在地面上，都會受到額外地獎勵或處罰。\n", + "也因此，final reward 可被用來觀察 agent 的「著地」是否順利等資訊。" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 281 + }, + "id": "txDZ5vlGWz5w", + "outputId": "bc284774-255a-45ac-dabf-3dfb5e1e5565" + }, + "source": [ + "plt.plot(avg_final_rewards)\n", + "plt.title(\"Final Rewards\")\n", + "plt.show()\n" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEICAYAAAC3Y/QeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO29ebwlVXX3/Vt1zr23gQYapBm7mwZs1EYUsUViVIxgBKfWRH3xyavGPAb1wUQSfY1DHofHkNdX45NonIKPihonomJQiAgqkwpNM9NM3U1D0xPdTc/TPUOt94+qXbX3rr2r6gx1zzn3ru/ncz/nnBp31bln7VW/tfbaxMwQBEEQZhbBoBsgCIIgTD1i/AVBEGYgYvwFQRBmIGL8BUEQZiBi/AVBEGYgYvwFQRBmIGL8hZGEiPYQ0cl9OM4niOjf+9GmQUFENxDROwfdDmG0EOMvDDVE9BgR7Y+Nvfo7nplnM/OjFZ/7ZUQUxufcTUQPE9E7qjynIEwVYvyFUeC1sbFXfxum8NwbmHk2gMMA/A2ArxHRM6bw/AkUIb9ZoS/IP5IwkhARE9HT4/eXE9GXiOjq2EO/jYhO0bb9PBE9QUS7iOgOInpJp+fjiGsAbAPwnPi4ARF9iIhWE9FTRHQFER0Zr/sWEb0/fn9C3N6L48+nENG2eP8jiOjnRLSFiLbH7+dpbb+BiC4lot8C2AfgZCJ6BRE9REQ7ieiLAEjb/ulEdGO8bisR/bCL2yvMAMT4C9OFCwF8EsARAFYBuFRbdzuAMwAcCeB7AP6DiGZ1cvDYUL8OwFHx8QHgrwC8HsA5AI4HsB3Al+J1NwJ4Wfz+HACPAnip9vlmZg4R/Qa/CeBEAAsA7AfwRev0bwVwEYBDAewE8BMAfx+3ZTWAP9S2/RSAX8b3YR6Af+3kOoWZgxh/YRT4KRHtiP9+6tnmSmZexswtAN9FZOwBAMz878z8FDO3mPlzACYAlJVujieiHYiM8pUA/paZ74rXvRvAR5l5HTNPAvgEgDcSUR2R8X9xLNO8FMBnkBrpc+L1iNv1Y2bex8y7EXVa51htuJyZV8TXdgGAFcz8I2ZuAvgXAJu0bZuIOpLjmfkAM99S8jqFGYYYf2EUeD0zz4n/Xu/ZRjeA+wDMVh+I6ANE9GAshewAcDgir7kMG5h5DiLN/wsAXq6tOxHAlapjAvAggDaAY5h5NYC9iDqhlwD4OYANcbwgMf5EdDAR/RsRPU5EuwDcBGAOEdW08zyhvT9e/8xRZUZ9/QcRyUDLiGgFEf1FyesUZhhi/IVpTazvfxDAmwEcERvyndB08jLEnv3fATidiFQH9ASAC7SOaQ4zz2Lm9fH6GwG8EcB4vOxGAG9HJMncHW/zfkRPIS9k5sOQSkN6+/TSuxsBzNeuj/TPzLyJmf+SmY8H8C4AX1axEUHQEeMvTHcOBdACsAVAnYg+hsiL7xhmbgD4HICPxYu+CuBSIjoRAIhoLhEt1Xa5EcB7EXnzAHBD/PkWZm5r7dsPYEccLP54QTOuBnAaEf1JLC/9NYBj1UoiepMWMN6OqOMIO71WYfojxl+Y7lwL4BcAHgHwOIADMGWSTvkGgAVE9FoAnwdwFYBfEtFuALcCeKG27Y2IjLsy/rcAOFj7DESa/UEAtsb7/yLv5My8FcCbAHwawFMAFgH4rbbJCwDcRkR74ra9r+rxEMJoQjKZiyAIwsxDPH9BEIQZiBh/QRCEGYgYf0EQhBmIGH9BEIQZSH3QDSjDUUcdxQsXLhx0MwRBEEaKO+64Yyszz3WtGwnjv3DhQixfvnzQzRAEQRgpiOhx3zqRfQRBEGYgYvwFQRBmIGL8BUEQZiBi/AVBEGYgYvwFQRBmIJUafyKaT0S/IaIH4tri74uXH0lE1xHRyvj1iCrbIQiCIJhU7fm3ALyfmRcDOBvAxUS0GMCHAPyKmRcB+FX8WRAEQZgiKjX+zLyRme+M3+9GNNPRCQCWAvhWvNm3EM2DKghCAY1WiCuWP4EwlGq8Qm9MmeZPRAsBPA/AbYimudsYr9oE4BjH9hcR0XIiWr5ly5apaqYgDDVfvmEVPvije/GzezcMuinCiDMlxp+IZgP4MYBLmHmXvi6egzTjxjDzZcy8hJmXzJ3rHJ0sCDOObXsbAIDt8asgdEvlxp+IxhAZ/u8y80/ixU8S0XHx+uMAbK66HYIwHQgomtq3LaqP0CNVZ/sQgK8DeJCZ/7e26ipEE1kjfv3PKtshCNOFWhAZf9H8hV6purDbHwJ4K4D7iOjueNlHEM0/egUR/XdE86q+ueJ2CMK0QBn/tky/KvRIpcafmW8BQJ7V51Z5bkGYjiSyj3j+Qo/ICF9BGCFq8S9WZB+hV8T4C8IIkQZ8xfgLvSHGXxBGCGX8xfEXekWMvyCMEJLtI/QLMf6CMEIo498S4y/0iBh/QRghEs9fNH+hR8T4C8IIUZNUT6FPiPEXhBFEjL/QK2L8BWGEUCmeIvsIvSLGXxBGCOXxi+cv9IoYf0EYIVSKp3j+Qq+I8ReEEULJPuL5C70ixl8QRgjl+Uuev9ArYvwFYYRQnn9TZnMRekSMvyCMEO0wem22wsE2RBh5xPgLwggRJp6/GH+hN8T4C8IIoQK9TdH8hR4R4y8II0Ri/EX2EXpEjL8gjBAi+wj9Qoy/IIwQiecvxl/oETH+gjBCKM+/IameQo+I8ReEEUI8f6FfiPEXhBEiyfMX4y/0iBh/QRghkoCvZPsIPTIw409E5xPRw0S0iog+NKh2CMIooWQf0fyFXhmI8SeiGoAvAbgAwGIAbyGixYNoiyCMEm1J9RT6xKA8/7MArGLmR5m5AeAHAJYOqC2CMDKEEvAdKGHI+OZv12Bfo5W73VX3bMCTuw5MUau6Y1DG/wQAT2if18XLEojoIiJaTkTLt2zZMqWNE4RhRck+LZF9BsLKzXvwyZ89gJtXbvVuM9lq46+/fxeuvGv9FLasc4Y24MvMlzHzEmZeMnfu3EE3RxA64pu/XYOFH7oaO/c3+3pcFfBliPEfBK0weuIKc2orqUnWhn3CnUEZ//UA5muf58XLBGFa8J1bHwcAbNnd30d/ZVBkFsfBoO57nl1XHXReBzEMDMr43w5gERGdRETjAC4EcNWA2iIII4NSe4bbrExfOLn/xZ7/sH9H9UGclJlbRPReANcCqAH4BjOvGERbBGGUCBPPf9hNy/REGf28269WDftXNBDjDwDMfA2AawZ1fkEYRRLZZ8DtmKmU8ep5ROIyQxvwFYTpAfX1aCrPf9i9yulK6tXnyD7JNpU3pyfE+AtCFVT0w9eDiCL9TD1hic6X4yEYw/79iPEXhAqh/jr+iecPDL9nOR0pFfDFaEhzYvwFYYQwPP8BtmPmUsLzZxRuMwyI8ReEEcL0/IfcukxDyhj2RPMf8u5ZjL8gVEBVP3u9pM9wm5bpCVuvzm1GJCgvxl8QKqTPkr8h+4TDbl2mIWXGWag1Qz7AV4y/IFRBVZKMBHwHS5k0zlGpvyTGXxBGiGGvFzPdKZPtU0obGgLE+AvCCCGe/2DpqLxD9c3pCTH+glAh/TYAbSPVc9jNyzSkVHkH9Trc348Yf0GogKqG+OvHG3LbMi0JE8Pu36bM08EwIMZfECqlvxagLdk+A0UZ9rx7X6bm/zAgxl8QKqTf9tnQ/Pt7aKEEZap6SraPIAh99/7Mwm79PbZQTHLLS3j+w/79iPEXhAoolRLYBW1D9O/roYUSpLX6Rx8x/oJQIX2XfSTbZ6CUCeRLto8gzGCqyvgIQ8ZYLSoaMewBxelIWrenuKTzsH8/YvwFoUKqkH1qQWT8h92znI6UyeQJK5L8+o0Yf0GokP57/kA9iH62w21apicdzeE75F+QGH9BqJAqUj1Tz7+/xxaK6WgO38pb0xti/AWhAirL9gkZdWX8h968TD/KSG2S6ikIQl8NgMrxV56/2P6pp0x5h3Sqx+H+gsT4C0KF9PPnr3L8x2rRz3bYs0mmJ+XLOwy57a/O+BPRZ4noISK6l4iuJKI52roPE9EqInqYiF5ZVRsEYdD0s/5O2/L8RfaZesqVd1DbDPf3U6Xnfx2AZzPzcwA8AuDDAEBEiwFcCOA0AOcD+DIR1SpshyBMOf32/prtMOlI6hLwHRilBnnN9KqezPxLZm7FH28FMC9+vxTAD5h5kpnXAFgF4Kyq2iEI/Wbjzv1Ytmabsex7t63Fmq178csVm/CTO9cly392zwbct25nT+dbtmYbnv3xa7F1dwNA6vm3Q8ZP71qPdsi4YvkTeOTJ3c79dx1o4ou/Xpk8OTy6ZQ++v2xtsv7WR5/CdQ88iS/+eiV2H2j21FadBzbswpV3rSveUOO+dTtx1T0bsNtqMwDcv35nz/cSADbs2I9v3LIGAPDUnkl89cbVYGZ87aZHsXn3Aec+T2zbh2///rHcQP7O/U185hcPodWO1q3Zuhffve1x5/HuWrsdV9+7sedr6YX6FJ3nLwD8MH5/AqLOQLEuXmZARBcBuAgAFixYUHX7BKE0537uRuxrtPHYp18NIArEfuTK+3D4QWN43oI52LpnMtn28t89hst/91iybTc8sW0fJlshnowNkzL+d67djkt+eDeOn3MQPnHVCrzlrAX4n69ZnNn/lpVb8U+/fASvWHwsnnHsoXj1F27B/mYbbzkr+l1ddtOjuGXlVjTaIU6eOxuvOv24rtuq86ov3AwAeMPz5hVsmfLaL94CAPhvL1yA79221mjPa/41WtfLvQSAa+7biH+4+kH86fPn4VcPbsan/+shnH3y03DpNQ/i4Ika/uyFJ2b2+dm9G/CZXzyMf3rTcwG4vfrfr34KX75hNU6eOxsAsPzx7Vj++Hb8t7MWgIiMbb/9+8dxx+Pb8ern9Oded0NPnj8RXU9E9zv+lmrbfBRAC8B3Ozk2M1/GzEuYecncuXN7aaYg9JV9jbbxuRmGAIDdB5poh9z3x/1mOzp+oxW9KuM/2UqXt0NOtrNR3nMrbuf+ptn+VshoxPsOyxwBeycj0WCy1S7YsnPU/WiHnATRW8n1u/cJk32i7VyZPG1rG3u5Titk5/KppCfPn5nPy1tPRH8O4DUAzuX0bq0HMF/bbF68TBBGkmb8mF+vBQi5AuMfGwll/IPYi1QGiRGd02f8VXMsmwRmBhEZZaKHJYOIijfpGmXw9Y66XZDDqRa31D13bKaOq/4f9OW2oQ2HwPhXme1zPoAPAngdM+/TVl0F4EIimiCikwAsArCsqnYIQtU0Y6NcDwjtkPvuPavjKy84CMzCbiFHHnuj5TNcsbGz2pUxfBi+3PQqmqM6u5DT70rdG6/nb90r12ah9kRhLs9uqz91DIoqNf8vApgAcF2sd93KzO9m5hVEdAWABxDJQRczc/+f7QRhilAedz2g5IfeTyOqjj+ZeP7RcmU8ODZiLZeVgdvIA6kB043QsMg+tkbeT9QDUjvkzFOR73tTWyqv3un5J/Ja1vPPbMtsPHENgsqMPzM/PWfdpQAurercgjCVKL28XguiH3W/PX/L+NfIrOrJHHmmPtknZDZe9eU1WLKP+xADo4q+yJR9OvX8Y83f4funxy3W/MMh8PxlhK8g9Eii+ceyT79/0422pflrqZ5AatT9sg+M7TPLh9Hzr/DYuuyTlGjOkXOiFaZX77pNocfzd3n4bZ7Gmr8gzBQM2adCz79hef5tS2Mu9Pxto+RYXqXG3g1VmEfd8w/ZvIc+2Sfx/Nv+7dLMoRKyTzh42UeMvyD0iDLK9VqQ6Mj9/FmrgK+Sl+Jy/hmPvijbxyczVO35dyVvVOj6uzz/NH7i3kfJPK2cJ4ROPP+QRfYRhJFFeX+651+F7KOMyWTTTPVMPFguMP6Wd6tIvV59WZ8ardGLvFFF9lH6xJR9+vF1fuoSWkmef95xLc3f6/l33vZ+IsZfELpE/abTPH+Kvcn+dgDK42+041RPledvGfVG233SRNf2pHqaef79N7bdHJMqdP3bjs4w8fw9+2Ty/J0BX3ObZLkz4NvlE1EfEeMvCF0SWh53LQhiHbm/50ny/JvmCF87r1xtl20n4u3s5eaTA1Ctp90Nas9+6uPOPP8Czz95espJ9Uy+D6sTdub5S8BXEEYX9dtVnvlYjRBy/0v5JgHftiX7xEZFeZp+zd8t+7gMaxX2qBt5w07zb/ZRI3GN8FVG36/5R7SSTsJ/3FJ5/uo4A+wAxPgLQpcknr9Wc6cKLbdppXrGc7mkuemF2T5mexWsBjsNY8BXYUlr/aCtXbf9VOTN9rGespyyj1VDyV5uHM8z6noqEeMvCF1ia/5jsewD9Nf7b1ipnoE1yCv1/H1uq8/zz0odwxLwVY5/MrLWI2l1Q6h53ck9yMnfj9oRkXj1ebKPJ7CuY6fpDoKpKuksCNMOW/NXAd9+ky3v4Bnk1aHnnyzXdqtC8+/HPfE91XRD6JB9ikf4mpk8rs28ef6uQV4FMYapQDx/QegS2+jWqkr1jI3JpDXCN0k/bBdo/j7P37G8EtmnG89fzVFvxVX60h5NcmE2jbDviS2T7eO4Tz7PP1f2Ec1fEEYP9bttJgHfwDvCtxePumEFfGukzm8aLtvjTM4dv5Yr7NZ1M710J/uYEd9+av6p7KM//eR7/op2jjxkj7tIzpcb8C3d7L4jxl8QuiQZ5GUFfBlZ49CLh5eWdzBLOttVJDuXfbIZJ8OS569Qe/ZT9lH9SJvL1/NPOti8bB+VfVVC9gm1NgwKMf6C0CWp5x8HfGtK9sn+oHvxXO3aPrbmr5d3cJ07lTbsFdGLmeffdTO99EPaaFQU8E2zbuJ1+THzNM/fIQ+lHUToXK4jmr8gjDBZzT+I8vw5GxDsRbNutkzN3y7prEsRLkOb8W6T9meXV5F33pPnb0lr/UDvNFXL9FnRXNiGPb+8g/s+u7aVPH9BGEEy2T6a7GPT6sX4256/leevGxvXE4ae0mnM2oWsAapG8+98H/su9jXP3xHwLcr2UYvtAVzGcS0Zzl6uI3n+gjDKOLxS32QuvRivhjfVMz6nZlxcTxi6h6+31aU7D0u2j2pGOoNWRXn+1tOPN88/85SVY9Btzz8n1VOyfQRhBLE1f2Y2DIpOL8arlRnha9X20U7oOg97jH8SCxjCPP9+ymY2uudvB7290zjGi9V3kSfllKnnHzru/VQjxl8QuiSdQSuMP6cGxf6996T5e2r7uDxNl/HXU0L1JxB7gBNQzeQpXRl/Nl/7OcI3DbZm0119Lc1M+tJBwFeyfQRhmmFr/onRd/yefTn4ZfCVd3Bq/p6pHKPtYHn+avkQyj4Vav66t68ut6zn3ywR8M1M5jKk5R3E+AtCl9iZKHnpe73IPml5hyjPv+aZyQvwaP5au/SUyZCzUwlWYYu66lDYeKku26dkwNfOjHJt5orB6MuN40mqpyCMLqnnb766fs69yT7m8X15/tE2Ds0/fs1o/ig3GrVXurl0ux391PyVKtNm1u5N9OoN+KppHHPq+SeF9srU9pHyDoIwuqjfbSPx/FP5x+4CupV92mF20o9khK816hTI1/wj429KPNl6P101M5fuZB+T/o7wTWUf2wP3T+YS75tcS3mDLrKPIEwz7Hr+adGv7LbdGi/XfmqQl56ymG6fPXmia7Od7eMo+dBHY0RWDaJOsHfpJWZikxhew/PPP35aPjvu4F2zc4XmNvZynaLOZiqo3PgT0fuJiInoqPgzEdEXiGgVEd1LRGdW3QZBqAJ7Anc9V7xf2T6uQUVpVc9ynj8bnr+Z6llmNGq32PJUJyR7WPdYpbn2gh7wLTuNo7qEvDl8Xd+H75jT3vMnovkA/hjAWm3xBQAWxX8XAfhKlW0QpgeTrTb2NVqDboaBneffyqn70q3n6kpxVPZP2XHduORq/naqJ7IebD89UWWmu0lnVB2W2rPRR+NvBnyjZaWncczR/L15/hlpLZ1BbDp7/v8M4IMwBbKlAL7NEbcCmENEx1XcDmHE+X+veQjv+ObtlRz7Kzesxjd/u6bj/ezaPokk0EfZx9Uuu7ZPWc0/zIzw5YxhtlMdv/W7x3Dtik24f/3Oju9RYMlTnaD2uPx3j+EvLr8dV929AUBUQkPx9VvW4Bf3b8SXfrMqd3Daxp37ce7nbsBHrrwPgC77ZEfu+j1/czt7q2tXbMIdj283tlHkPV0V/Vt84Vcr8en/eih/oy6pbCYvIloKYD0z30PmbMwnAHhC+7wuXrbR2v8iRE8GWLBgQVXNFEaEzbsPYMueyUqO/YsVmzB7ooZ3/OFJHe0XJhkiphF2V/XszvhfsXwdAOCM+XNw9xM7AGQDvrpxbTjy/NNgpdlRMBcXIfvW7x7DomNm49oVTwJAZ/coeULpwruNd3l0y148ufMAjpw9rh8SAPAv1z+C3Qeip8GXPWMuTjv+cOeh1mzZi9Xx3/963Wlmnn+8Tchuo263xxfXedd37kjeF8k++v0ouje3P7YNeyareeLtyfMnouuJ6H7H31IAHwHwsW6PzcyXMfMSZl4yd+7cXpopTAPCsJpMlOjYWe271H5Wbrg+9N8+WreDlAIC3vT8eXjuvNSw1YLOUj11icGcrzdbh8iVYtmtZBX0EvDV7uCLnn4Ubv7gy/GXLznJuK9Fge5kO20Vwwz4Zjz6Qs8/zN1O3yb97D6W/d4Fc/oE1W968vyZ+TzXciI6HcBJAJTXPw/AnUR0FoD1AOZrm8+LlwmCF98MWf2gHXJXNVZsvdiomNmn/Pkw/vHrT8+2US0M+ELXuDvz/Ftt7jpYrWbj6qqqp9YOJXMRmXMk65JVXued7fDSffSnIvu8rvakAV8/RbV9OvH8Q2b0IczhpBLNn5nvY+ajmXkhMy9EJO2cycybAFwF4G1x1s/ZAHYy88a84wlCyNUFx1zad9n9otfoc1PrQeyjdVswLWRGEJjeX6qlm+0ACgq7MVtF3LL31G5nsx12LVklgekeUz3Vkw6RuVy/lrz/jdDT4UXZPuY2RZp/Syvi5yMj+9gxAE8H5jsvDaPn3yXXAHgVgFUA9gF4xwDaIIwYzN1552VwDaQqQ5oh4vL87W27a1vIkcere392bR/d2DScef66sdPaiOw9dck+xsCwkJOYQxHUU8A33Uedj0DGfS3r+RsdhpbeGu1vyz6+9pjnybuiwoBvyM73LqInv9xNumZKjH/s/av3DODiqTivMH2IiqZVJPt0KSnZXmPLGj1rbttd2zl+7NcNrl3S2dC+HamhuswRsrk8W97B3Nf2/NvMCFDW+Kfn7RRT9oleAzI7hbIF6UzZx8rz10o9qPXu9rCxXd7XWTSNYyeyDzMjCKpJypQRvsJIEP1oKzp22J3xt+vh6z/6/nn+HGv+6bJseYf0vEWyj6n5uwZ52bKPWQyuE0NuP6F0S6DJPkmHm/Gm/fubHZ7p+avOJB1X4G5rGhso9vztW5R5Eugg4KtiPlUgxl8YCSoN+DJ3WXxMvRZ7/t1r/tGPP3AGfONXre15Ad/Q8vzZ2c70vZLD7LEBZUnKO/QywhdpwDcgynjgSVvLav6h/iRkPhUBxbKP65hFZJ6utK+o6P8u0vxLn6ojxPgLIwFX6vl3Z6DsUaG69t6vmjnqx6/rvnZtH9Pzd2n+0WubLUNY4Pkro68fc6o8f2fAF1ljnbbLb0XZ8rT14G4228fd1oyx7+CSegv4iucvzHAq1fzDXrN9lOfff9mHnZ6/NZOXduy82j6ZgC/np3qmxl/z/Dt4QlIt7srz19qZyj7pyGaXPOXDK/toqa925la2PdbnDqx/Js+/g4CvivlUgRh/YSSoWvbpzkBFr658+6zx797zDwhmnr81yMsc7JQf8DUzX/ID0+k8AmbAtyyqzd0EfPXzpHn+0WdXp5WXjpoJ+OrGX52vQMvPk8eK6D3PXzx/YQZTdcC3F8+/jOzTbb+lfvyG7BOY5zBH+DpkH5XOyLbn76rnn+1Iug34Jtk+XVy7fh4l+ygjyMg+geQb//Q9M2uZPaYEpL8W0cn32ZPsE6KyPH8x/sJI4HrU7xdthxEsgx3wzUs97GWEL3lkH1d5B+c0jqqdjlTPbGwifd+w5ino9DqCHgK++nnU9ZK2zjaaeXMXm5q/u6pncg89h+nl+8wGfMvLPiM3wlcQ+k3I1dX2ico79KL5Z9fZi7rX/OM8f8cgLz2FU+HK89e3swd55dWdcWn+HXn+6D7ga3r+0auSu1yyT14JClPqMr39svX88+K9RbGoXjx/FfOpAjH+wkhQpebfrexjBwvNddY5evD87do+NSvPv3AOX227Tmr72HMH2+cqIp13oIuOVbsMe0Sx63+hvOZvBXytbXxfU56MV3R5eSN+y3j+VaV6DqK8gyB0jEui6Bdd5/kXFAPT6am2D9m1fdJ1QAnNX5OnTP0bCGEbtfyOpLM8/+7LO7gCvvo96Czgq7238vzL1vPPLi7fIebn+ZeRfaZPbR9B6Bi2DFc/iQxCL7JP8b7dtJ050qR9tX2S0gTxwcdq5NH8Odk+U0445wnFdazuAr69yT6Ble2je++K/FTPdJ0+JsLI88+X/LOyjyUl5ZEn+xTdTgbE8xdmNlXX9ulHwDd/225kpejVzvP31fOfqNcKp3EsGuGrr3fV8e+qkyx5b+3BWIogyfZR22Xb0XDEOlzH1YPXbW3cg2qj73/MzuvXPxXdk7xUz6J9RfMXZjz643q/6Tbgm2r+xdt203ZlGKI8/3R5traPMv5B4TSOmdGuJUb46nQij7kC0mW2j86TlX1UANmWr3xtVbgGrql22dk+fs3f/tyB7JMTVJ929fwFod9UFfBNBvx0JftEr2WeSLp5alHHDwJ7MhdzDl91XybqgTvlUTPCoeX6Z/P80/d5ElIZUj293PY+o6iyfZJBXsi2266h7ztuyypPbT+9+TV//30qGvWckX30p48iyahCzV+MvzASMKu//nYAroyZshSVBHBt283xvbV94kOqshITY7UCzd/O88+fxtGVNtrJfepEFtO3t/fJlHdwxGjyZB9DytIsdXQt5vfva6l92frHIgPeU56/DPISZjpFqXjdUpTlkUf1Ad/o1VfVM2179Hm8FmRqyQO25p82JEr1tNupyz69af5sGdYivJ4/WZo/XAHfcpp/06q8mmT+FHTk9ovVoZ8AACAASURBVGL2tNWFfc/MSWhyd5XaPoLQ6fD7To/by4QjZeIFvXQumUFe1mQuyuBPjLllHzO10Ty+7ZXqH92af+eef9l9zLal7/WqnmpdR6me2rYta8yCXvQuakM52cc4fqfZPh3JPhLwFWY4iaGtzPPvXFLq5Gmkmz4rNf75g7yUs+8L+Pqnccwapv5q/p3t4/P8lfFLR/h2X9VTn2c5KnGt3ud/l/byjgK+OfuWKu9QkZUW4y+MBJy89tnzNyYB73DfDoxbd+UjoldfbR913sTzr7s1fz3rxhzkVa6ev05n2T59kn1KeP555R3yAr6cvI/b7Pn/yhvhWyj7ZDx/7X0Jz180f2FGU5nm34EHZ9NZwLejQwNIjWe2tk98zNA8ttfzN2byMjX/KgO+iWHtU8BXpfswHJ5/bp5/+l6fc0GXfdLy2MXHsD8X5vnnyD5Sz18QCqhK8+9kwI1NXm0fm+40/+jVN8jLPubEWOAO0mqTlNsVLvMGefUa8O00nuILomYCvpx9Ailb26dp6e36UxHQgeffS3mHDoPFovkLMxrby+3bcbl745/m+Rdv212ef+r5G4O8PNMjTtRr7qqeWtaNmeafrWmkt7PX8g6JYS2d56+/12Wf6FUN8mKH7FNW89c9f/1JSHngZS+vE89fAr6C0AOdeNmd0MmsSjZVp3qmef5uz98+7UQ9KFHP3/T8s15p+t6p+Xfh+fct4Kulemby/LvQ/F2ef9l6/vqnos4tz/MfZFVPMf7CSJB42V1U38zD1F8727ejgG8XnZbaJSAyMj58nqA/2yd6dQV8Oy3v0FHgOvH8ezP+ScA3KezWWaqnkecfuj3/juv5d5Lt04Pnz6Pq+RPRXxHRQ0S0gog+oy3/MBGtIqKHieiVVbZBmB5UnecPdF7iQRmAMnv15vnD8vzd24/XA6dh1MssdFPPX6ezPP/4vCXvqymlpO9r9ghfdszkVbK2T8bzV9sUjvC176l/XdG+5tNm7q6osrZPZVU9ieiPACwF8FxmniSio+PliwFcCOA0AMcDuJ6ITmXmdlVtEUafTksFlEX/If76oc246p4N+NY7XlAqvU4VSqtK8089fzPdz9e2iXoNQGQIa0EtPY5qL2dn8soaNU3zd8QPOhvhmz1mHr74iz2NI3P2CSRvGkdD9rGf9JL/q/y22ktN2aczz9/oOAYY8K2ypPN7AHyamScBgJk3x8uXAvhBvHwNEa0CcBaA31fYFmHE6aSCJgDsPtDEQ5t247nz5uCTP1uBS847FUceMo5P/fwBvPMlJ2HeEQdj+94GPvST+5J97nh8O256ZAsa7TAxpO2Q8dlrH8Y7X3ISjpo9YZxj294G/vaKe0q1Rxmgp/ZM4nPXPYKPv3YxGq0Q/3jNQ/j7Vz8Lh0yYP8UtuyfxtZsfBeCv528zUY8eCb5642pcct6pyXI9nVG/f/983Uo8Z97hznYCcJaKUJ7qz+/dgGY7xBueNw/XPfAkntoziQvPWmBsqw51zX2bcO2KTWi2QzyyaTeOn3MQfrFiE8591jF469knaufWz5OVfdR1f/oXD+GPFx9jnGt/s433X3EPagFw6RtOx5j2eOQK+NYCwgMbd6Xnixt759odeNs3lqFGwCtPOxabdh2I7qVD9mm2Q3zwR/fizrXbM/dJZ1+jjQ//5D6cuWAO9jfb+IerH4yvJzrvrY8+hXue2IGn9jZw6jGHYsOO/Wi2Q5z/7GMrzfOv0vifCuAlRHQpgAMAPsDMtwM4AcCt2nbr4mUGRHQRgIsAYMGCBfZqYYaR5vmXs/4/vmMdPnX1g/jJe16E7962Fi88+Wk4c8EcXP67x/DMYw/FhWctwLLHtmHZmm3JPko6aLUZyhav2boXX71xNZ557KF4/fPMf9M71+7ALau2lmx/9PqP1zyEH9+5DktOPAJzD53A95etxevPOB4vPPlpxvbv+8Fd+N3qpwA4Uj09xuBFTz8Kn7vuEfzsng2G8fcFfNdu24f1O/Y72wmo+xBgUnsCUEbye7etxYFmG2943jxcsfwJrH1qX8b46+d613fuyLR3084DpvH3pN3ak7lcfe9GHDbLNF3rtu/D2m37onOdcwpOmTs7Weeq5/+q04/Dz+7Z4Dz3zSu3gBm4aeVWhMy45LxTnQHfDTv248q71ifLjj98FhYdcyhufGSLse2KDbtw77qd+OHta0FEaIeMQyfqmD2rjjBk/OyeDfjubWsz96cW0PDm+RPR9UR0v+NvKaKO5UgAZwP4fwBcQR10Ycx8GTMvYeYlc+fO7aWZwjQglX3Kbb+v2UY75MRwtdqhke8OZHVi14TlebnqeTpztv3mOYPYCOjt0dm2t5G8z9b2cZ/j9BMOx2uec5x3snE9u0WRN8K3HTLGrQBDUgI75KRsgW8OZAbwzhef5G4s8gdO5Xn+ANBo2e3OtjFdpwV84/v/p2eegKMPTZ/k9PbPHq8nbVBxEfvqXLGVz77puXjmcYfCRi8hot5/5o3PwXg9yBTbM84RP6kNpezDzOf51hHRewD8hKNudxkRhQCOArAewHxt03nxMkHw0nHaYGga22Y71Grh5Bt/PW0wz0Dn1ZC3sVMKg4C0HPPs9naqo6u2j01A0bpssTZN9ikcUaq1gRlj9QCYzLYr1DKFMvMEaOcdq/v9y6Z14XYaqsKu5w/kd7x56asq4FsLyLiP+i72NTfbYfb/zmG0AyLvU5lNEETbtkP/LHKqYxhKz7+AnwL4IwAgolMBjAPYCuAqABcS0QQRnQRgEYBlFbZDmAao31lZ469sQyMx6OmPTL3aQULlTTatyo+AOzDXifG3BxNFP/z4HI5r0q/TzvbxeYLK+PgCjHoJ46J2qraO1cxz6fMf6PfT6flz1CHZTw8K24D7vttU9knbkmv88zz/uMOpWVKajn3NjXbofJqym2B3KHnUiBIHwHcpqgrrKGr+3wDwDSK6H0ADwNvjp4AVRHQFgAcAtABcLJk+QhGd1vZJpJ1Y9mm2Uu9N2XZ7YFDylNByyD4uz78j2SdulzL+QX7Hoi8KSgZ8ieInCo/x0+es9bdT6/gYRuBUb2tUETN9CnB5ryEzCBRPLJ89l935+jomu7AbUFTC2fysX7Ly/IMcQ123dLVmy2H8HbJPLSgv0dQ0z9/3nbQ0ibAKKjP+zNwA8H971l0K4NKqzi1MP7qXfVKJxza2tgFR2S1Nh+zj9PxzSgoUtT8gSp4cXIbTln1cI3x1KE4HrZFL9lFtKFOKQH+f1fx16Uz3/J2yDyLPf6wewGX97fvvC+antX00zT/n3ufNnKUbVK98FsTymfY/lJnG0ZEma39PeQRB5Pm3Q3+qqPr/GEXZRxD6RqcBXzuoqxt/b8A39kT1J4LcgG8HQ4Lt6pG6cXE9VZjG39S7XbqyXvM+b3auIqnKHrlatyQQM+Cbxix8sg+IMk8PCvvJy9e0dBrHdFleFc/cgWvak5fPqNYs7T7S/M1tQofRjmQfb7Oy5wjipyZPp5ckB1Rk/cX4CyNBp7V9lJHSNX/bkNsjWNW25mxP8avjtB15/tZxzIBvkeZvepTk+NUq+6AMio7+qUiqMnLt4wFGuu3xBXx9BeL6ofmn2T7adeR0vHllqst5/mY5jUY7zFT7ZGQ7vCDW8csQBGnGly8Ir/6/pLaPMKMpGoFpkxr4rOefdAyW9+hK9bT3YYchAYofzV2GvhPPvyjPXwUF3QFfvc352SN2wNcOYqrOS/f8Xbq1aoLS/F3Yna/vq02vvaTsU3D9QNSh+CSaMp6/qy6S0vHLoALOvngJkD5ZjmRtH0HoF51MnAIUBXw9so8j1dPeRzdQem14O0ioUw8oE/C1NXMbe5BTUcBXrXcFfO0257XV9vxtI2mmd6Zt9RncgLJB4+T4VppjJ55/nuyTDXhr+yU6ut/zrwWmB99suUt4uGWfDgK+QVHAVzR/YYaj18/pLeBrr7ONf1YOsg20S0IA/AOvonWUav4Ob9l1TYbnH9i1fRzn0D3/Atkjr6225m8bSf0JJi/gm3j+BNRzhHD9O/B9tcqbLp3q6QjOKvTyDv7xEua6KNUz26E6ZZ8+5vm3xPMXZjr6b6xsjFX9MJ15/pw18kAqAzUd5QxcVR91zb/Y8zdlIz2/22XHdINQprZPYvwDh+yjvW+2izx/0xO3pYy21okZOf8eg0tEGPfIPlF7sk9ZNur0huffUZ6/vl8J2SfIyj52yxic+V9Ug+zKUIvLdEcBX/c2zXZ6D6tAjL8w9HQz25ayDbqOnwZ8zXUKZ3kHq8NwDRgC8h/Na5SVfdphtmPRyeT5B/mef2IgtY7GdaxWGGbaqn82ZJ+Qk8FIyfpEttLeO2WftF0+2QcwO+Ai2ccc4ev/P8irVKp70/kBX1vzz95Tu8Oz5aI8DNnHG/BVbS11yI4R4y8MPfpvo+wgL2d5B0u68Rp/R1A2Dfim2xuef46Bq9VSg5wETK1sGZtswDdd57IFhuyTjU4ax7XbWjcqYGrGOExz3tP90+PkBXzVxyAn1ROwplX0yT5WPX8gf+auTKqr9tko7+AN+CLr+WduadZod1TeIQn4lsnzF89fmKF05flb0o5e28eX6plsmzPC1zD+1kAsHzWiNGZRMuCrdwh2bR+XDGAEfNn0dk3ZI8y0dcxT48YV8DVkn6Qz9ZdUICC3vk/D0Pzd361dz19dh4/c8g5K8yfyxj7seECj5ejcHOfpKM8/PkepPH/x/IWZiqH5d5rnr3T8tuZpW2MAFO5UT/M1W98xop7zC61pUowRJM0J+OpepV3bp8jzj46ZrjMDnpxpq8/zTwO+2XaFoXk/7Q5HvQsKNX93J6XjquqZm+3jkGjUJSfedODX5+1pM1th6JzNxSn7lC7vkD6lFef5i+cvzFB81R7zyBvhm8g+lgFRhsEw/paBLjJQvnW2odc95zKevyH7ODV/FfBF5pi67NGMc/d19Dz8bgO+0fbIHMen+atzlgn4ptk+2nVonUamBIUj7VRds57t00nA1+X5O2WfsoO8SJV3yPH8RfYRZjrZn10xrkFe2RG+oXMffQCR/bTgrT9TaPzNc+g18F3GXz+NnUJYJPsApiHVj+5K9dSzf8oGfCOPVe2TvY404OvW/A+Oa+Y3HBJb5toCdY1uzd8eROYa4ZsY/7iNtU7z/K1tXHJNx3n+8SAvCfgKgoduPP80qKu8/DS10l5n4xrh6/JudYqMfybPX/vRF0lZdm0fIPvZln1MQ2xKOXZQUq/fk8nzt0f45nReRoeja/4O43/IeDrfcLpPZrPomlTA1706E1NwpXqqDs6o6unxqO3AbcPl+TviHHllop3nKPD800Fe4vkLMxS2Kk2Wwdb1G46Ary9jpJUj+/gePMp6/mbAV7U1/1rs2j5A1hCmtX3IaDdgjfBthw7Zx+35h8yZUgfugK/f8w8IGK9n783B8TyZpVI91fk9t9juXFwBX3XNRj1/r+ZvxRec9fwdVT1z4gg26ukiCpa7t1Ftldo+woylK81f5fK3NNlHy7FXy1zoso9djdMX8M1L8VOP94A5wtc1dsCFbYyArPSjPqvtfPPhthyav2sEr2qjLYHoef7qs6tGUar5u2Ufl+fv+27V+X0ecGaqSYeXroLcZTz/jOzjTPXMGu3OAr6EGrlHRyvE8xdmPC45oew+neT5K1zZPmUDvi7Pz/T84+NyNp7gww74Rsusz4HZDsML15a32oyaJfrr2T9mZlXWSLbtTkx/ArDOqdqZq/l3EvB1rkW27LR1GEPzL13eIf3cbDMYZqfJjMoDvonmX5GVFuMvDD0hu9/nYefyN9tZjdqeSUrhKu9QNuDrSvk0NP+SAV8dO88fiKpl2tsAqZdse+GqXZHsYx7fNs56W+14gz7IS73awWx1zqihHs9/Ivb8W7rm774PiefvMaxlZJ/k+rWAr+94drZPoxXGcQNze3fA13lI7znyAr6S7SPMeNghJxShtks0/1aYCbCW8fxtqahImvAZ/4zso/3oi55mojx/e6F1fjvPX780XfYoCPgCMIy5nRWj5/dHbfc8FSnbj2w2DpB6/obm79G+CwO+BbJPyNEoa3VNgKqn7z6eq7wDszk+wlXSwhXw9Z6D0jpMRZ6/5PkLMxZX/ngRznr+lsbuC/jqmr8tFRUN8nKVeagFZGjk6rh59fx1XNUiM31BEvBF5pistavZDjPGZCxwG8+QHdk+VoqjK/Crzqnanuv5d5Tn79P8LY/ckeevsn30Eb7+8g7uwm76d6t3egrXvMBF9YPy6vlLSWdhxmNq/uX2ceb5Wxq7byYul0FKZR/3+ZSxcHr+joAvsy6X5F+LmvVJx5fq6Qv46gFPV7xAP55u/G1vNgzNLBd9pLJL9vEN8nJr/pnNkva5rllhd7iuVE895gHkF2HLBnzZCBoDqqpntsF2h+IdS0DaBO5e2UdKOgsznG5q+6SjeDXNv2S2j5HqGdpG232+RPN3SByR5m8dLzSNrI5tvCLN3zxmVvM322GnXap2RVU9swZKX8ZapxQE1jSObJYy9tUo0lM93bJP5Pm3HJlVNq4J3HXKDPJKZa/QmOzehZ3nPxnHJczxEHCWYrY7lLyMovJ5/s7VPSPGXxh67AyUMtief5Tnr46RL/sYk7koA10wICsN+HpkH8s7zivvYHdKAWWNSjb7x8w2ygZ8lezBDl3a7EoMz9+aSMY2Vm3tCcbVSRMI447Cbqnm30FVT/dqR8DXXK97/vr1+z1/02OfbLUBmN8twz3mxNWxujCrejo30QoHiucvzFC6q+oZvTZyZB9/nr8r4Ks0fzdFnr9qt/Lm8rJ97E7JnsxFLXN9dsk+uucfbZttnyHtWAFfu2PQ29tyxEfUOdW5etX87WuzKc7zN4O1taSjdB4uI/s0nJ6/22O3j5lXQqIWmLKZD/H8hRmLbhtL5/lbBp45NaqdlXdQbTCfAGyK8vyZo7arNuSVd7ALzkWeeX7AN3+Er5mjnvFOLVlJv9YgICvV09SofcY7Nf4FtX1KGH+FT/ruZIQvoI2JyJV90s/K+Nt5/q5Areupyn2O6Mmjzf6Ab9ExeqUy409EZxDRrUR0NxEtJ6Kz4uVERF8golVEdC8RnVlVG4TpQXcjfLMGfrLZTtblzZ3qqupZ9ANVhsHOnAHSaRz1+v959fztTika4WsdNPMkoLZ1aP4wg5V2EwNb808GoinPX5d9zI6l4egoo3OmenWe5q+PtSjq173Gv6i2T2ga7lqh7JOdwxewA75uR6BMtk+gxRzs1FkXozjI6zMAPsnMZwD4WPwZAC4AsCj+uwjAVypsgzAN6CXPX/eiD8Tv2yHnTgaiG6R0tq3886dSguvHHmm7l//2sWRZXnmHrObvGuSVPYd+fiMoy+a8vS7PX2+2Hp+oBWSczB6U5CqCF20Xt5OysgwAzBoLUAuolOyjsJ9+FHkB34c37caqLXuMoLky+nmev36PJpvK+Oupnm7Zp4zmrz8l2qmzLkYxz58BHBa/PxzAhvj9UgDf5ohbAcwhouMqbMeMhZnx/ivuwe9XPzXopnTEgWYbX/rNKuzc3wRgevvv/d5dWL1lD25euQUPbNiVLH9qzyTe/Z07kn1cxdsOxJ5/yPnG/6m9k3jzv/0eb/36bcl2Ycj4l+sfwTmfvcG5jxpE5NL86zXC2m37cOk1DybLjGkcQ8a/3bga96/fCQB413fuMPZ3DfKyvdZMPX9mfPO3a7DkH67Hk7smLc3fne2TdBxWnr++tR3w/avv3ZW8V/t9f9la/HbV1qidHtmnHgQYq9nGP7OZdc3W57hheudSCwiX3fQofnZPZG7+/qf3YcvuScOg50l0arnT89c1f/Ti+acxDGZzRjgXIyf7ALgEwGeJ6AkA/wTgw/HyEwA8oW23Ll5mQEQXxXLR8i1btlTYzOlLK2T8+M51+P3qrYNuSkfcuXY7Pnvtw3jzV3+PS35wF66+d6Ox/o7Ht+PjV63AV25cnSy7d/1O/GLFJjy0MeoQ7MlcgNT4P/bUPtz0SHRPPv7axXjoU+cbx1+9ZS+WrdmGm1duxba9jeh4IeNrNz3qbXNenr/68U7UA7zp+fMAqIAv4mMDn7n24cRgPbhpV2b/okFeyi7pss+963Zi657JTLtcHcnbX7QQrzztmGRf9WrHA2y5bOXmPcn7VZv34P1X3IPPX78SP7w9/YmfdfKROO9ZRyefzz/tWDz/xCMwVguMzrlldcjPOu4wfOw1i7VrNht+8tzZeNXp0bGS+xAQJlthci/3Trbja9TiIiVkH31dmu1jaf5xLOE/3v0H+NTrn53sax/LdXz9Vb/uP3/Rwsz2QxnwJaLrieh+x99SAO8B8DfMPB/A3wD4eifHZubLmHkJMy+ZO3duL82csaRpjiWF8iFBGZeHn9yNq+7ZgFsfNZ9cmu0Qk80w0fABvXqnnZqZ7ncgfnzfsnsSF3/vTgDARL2GWWM17zSMSioK2Te2NyIv1VMd+5jDZuGzb3oujjxk3PD8W2EYS1HR5/FagIteenKyf+T5uz19hUqn1L13vePLl32AD7zyGXjZqZGBboacDEKLPH/TC/aVYbh55Vb8+M512LTrQBIkDYhw2KwxfPy1pyXbfXLpaThkoo7xWmC00X4aO2P+HPzFi0/S2m2e77BZdXz5z56P+UcenFy7MuzqWC1toBRZRr90Pf9WVvaJsp6iY7xg4ZF469knJvvquM5hS4T6/+iFZ813tqcK6r3szMzn+dYR0bcBvC/++B8A/k/8fj0A/QrnxcuEPpMOcCoYQjpk2FLApJX90myFaLZDy3CY1+rSUQ9onYVCPcqP1QK0wux6XSrKQ/1AXbKPMjZKmw6IosBp/KtX50gNFhtSRkAEsvoU+yxKWtEnczGMv9Yun4Q0Ftfdb8aFzNTxdNvT0spk2BxopfdPecu2zq6uR7VZj6/Yge7MwDbrc3LNmjHX8/n1Vz1dttjzt/P8s7IPWMli9r7mMV3nyCtR7ZLIKrL9lco+GwCcE79/OYCV8furALwtzvo5G8BOZt7oOoDQG3qO+yhhG4GM8W9zbPyzgcckndNxyQcck34rI+vKSAFSw1yU7VO3HuVd61JjZeb5qycSVXa6HbJhBNyev/s6lHcbak8Sdrsymj+Z7dPLX9es0g/6SGkb/UlM9/z1c+htGatTrudfNLZBPe2kg7bSffSCfupYGc3fN/rWktmcqZ5IZTF7Xx1fob/oNXtuV3B8KD3/Av4SwOeJqA7gAKLMHgC4BsCrAKwCsA/AOypsw4ymObLG32zvZMv0yBux4df1YrujcxmoSYfnrwyeaxRqtI96ksjvAALLwOuoH3sizRAZso/qYBra08yYNvuVq56/7fvbXnA7I/v4Pf+a1XZ92kJb9nFNZq5QnRiQdthqT91D1jubRo7xLxrbkOf5t6z/Bz1uks594LyMuEBb+jnx/A3Nn5NpLs19rWM5DLddh8l1TUXH6AeVGX9mvgXA8x3LGcDFVZ1XSEm8YU/d+mElY/ybtucfGoZS3ydX9nF4/mM1v9GO9olln4L00DzP3zauQUCm56/FK9Q5bNmnqFTwWD19qgCysk8tR/NXBmw88fxT796WfRraU4HNAYfnTw7PXzXF1vzt2FRGniK7wzOPrVfVtGXAKLceme1d2JOypJ6/Vd6B82dFc32OluWtc3UWzmb2jIzwncaMrOffKpJ9spp/y9J4XWl4Ts+/rmQfj/HX9Hj9kLZMlDuZS+Lpph2EPrJTtSuKZXCmPbrhUmT1b9ObjAK+nFmvb5Ntnyb7sJJ9zG2becbf0PyV8UfmOHpn2LKku7GclFSf5q/PYqauLYmfJMfXPf982Ydg3iP15Km3TZW5KJJ9XB2Mb0yIb3axUczzFwZMY0QDvnZtG1v2OdCM5lQ1g4UlPH+H8S/S/JURs/edqNeMz3m542QZ16SUL5uyj96hmZq/K9XT0r9t2SeE5fnrRtVsX2qMY8OpTXxjDzBrtvy1aHTZp6HJLYAv4Eum7NMKjSceu532PbCvWdfqE81f8/ztgG9erX19nepEM9M4skv2sTsD9/Fd1+ObY0A8f6FjVJqbr4bNsJLV/M3P+xqtzHaJ5q+N4rU50Mx2gmO1cp6/va8dI7ClHR31Ix/XZR/N81fHbrTDRGIYs7JzigK+iResZfs0tPs2VsuRfZQxrqeafxrwTbuZiXqQ6/nvb2Q7V7WvK+Bbd6R6jtWD5Np8I3rta9JlHHt8h0vzLzPPbpEeXzbgmyftZOU3dymHURzkJQyYkZV9rPY2LOOvBu6YXmOx7HOg5Qr4xobZE/BVhtned8Jj/PM024znH5rt0j1/vT2ugK99Fjv4yWxr/mZnYrY9ejU0fz3gS+k168FgG/sJLdo/e/6a1hnqjkmjHZWhSAOi9rEszb+elXGSgn4tNuYXDrTS1HkBV7W8UI/n6OmqKLXTeZykE8ouzxsR3G/E+E9jRlX2KXpScXn+mVRP7RDqt+OSfZTRTNMxzR9a6vmb+2Y8f3LvD+h59KnnrxdIS2WfVKe3PXVfCefkOupm59PuQPO3M5Wa2kAuvaTzxFgt9vwzlxhfR3aF2ls/pT7uwf4Ox2vp+TJxDuvYtudfszx//dhGnn+QbZNxHnJvM2aUd+BSef5OY+71/LPyXl47e0WM/zRmVD1/29O3UZ6/K8/fpfnPivV5Wz4CUiOuRm/aWn6q+Zv7duL5qx9vGvCFU/bxa/6ZQ2ZlnyAr+/g1f7dUkWj+RsA33T6Sffx5/q7O1c6w0dsxVguM71rJPj7PvEjzDwKzppP+ZEja/nmDrKLl6bpZ2v+Dq6RzYcDXcY5OA77i+QsdM6rlHYo6q8Tz1wyHneevyz6zxqJ/c5daYWeMqG2L8Gn+rvIOtuZvB3yTa2ilxr+e46kDxaNd88s7mPtm8vztgG+83Xg9QDvkJJZk4ypQRpah043jWN3U/FvteHBb4vrbxzI/j6Kg1wAAFGVJREFUq3vklH3aoZFJFHI6d7G6tV7PH2mev/7/kCnv4Az4msfK8/wzxt8X8K3ISovxn8Yk3nCBJz1sFBn/vQ2H5p8Y/7gome75j5nevI6efglkPX8f9kjMvFRPtUg3VvaMWOoa9Po+9v46djA0kX0Sz998gjIHebklJNWhGXn+huYf3RuXvOMjSAxtdBzdkGU1/xBjtSDZp6iYnSvg20qMv9n5tUPOdEQ+f5p0z1/733HV8+9uhK/7+oLAPam8eP5CxzTa01Tzn3Ro/nF8o9EKwcyGl59n/MetLJmePf+cbB89I8c1mUxLk2rysnMAf3kHZVzD0Jw8Rm9XxjvNyfPXz62kLpe840PvpOyyCX7N322cM3GO5H4iaas+F4Hezmg0brpdfEBvu53GX8/2iWWfovr9eSN8s9uar/byfiPGfxqTVrocLePvm1hdoTz/UCu5oGv+tlG19XkdWy7J6yh0xu08f/J7/kV5/opmK9WpzcFO2fN7vWAj4JveR71Z2Xr+6hip5p/IPgElZ+vK+GunCgJT1hhzpXrWtFTPAtlHT50FYtlHu6X7tNTTlmao01RLd5v1bB/9f8f+bl15/uoJJ9nHlQDgiWn4YgEyyEvoGFsKGRWarTDX21GaP5DN8tG9VkWu7FM3pQP1Y3cV2NLxBXzdP/boVRmPwAr4KhptTjpse5BX9phuL1hdRysu7KauQz+VL4VSr+2TBHw1Y6aedlyBcx96M2tkyhqZgG+LY9nHbRzLlHfQ76n+f6J76erVN45Az/YxPX9z+1bIzvpAejudAV+P5u/NAhLjL3SKbRhHhWY7xCHj/rJTeVk+zTZnKnr6cvgBrSyCpfkfPJH/BOAP+Pp/qHr9Hz0HXWFo/jltBpBx/e3rUCUjDornym21Q+/goozs09I1//RUqiPpWvaxslnG6w7Nvx744r3Zp5265flbt8zw/NucKS+d5/m7ZED7u3Vl+wCm7p+f7WMt93UKIvsInTLKmn+R8dW31V8bDs+/HpC3fINKkaxZP/aDC+Qff6pn9icVJoY01fxDds/d69L8XdhXM249wSjvXE2U3tIMlU+njmrjRO1QTdPnvlWebyeev2649BRKwK35jwWUXJz9xON92nFkEgGm8dc9/8JsH0oN84SR6ml+J612VvYBzJiKMwHAE/BNOgXx/IVeGdVsn0aB569je/4th+ZfC9zzyALZtDv1mD9rrJZbBsBn/F2djAq81rXMIl+2j0vzd+ENfsbtUN75QWPK+IeJ0bFtiWmQzRIOUXkHJWnFsk9Hmr/t+ZttbsWzhgGa5p/sax/M/DhuSV22ITZkHz3VMxnA5ZN93J6//Z20uYTn78z2KZB9Mpq/s5k9I8Z/GpPUNC+aGXvIaLbD0p5/o+WSfezSwH7jn2xjGf+xWpBrgDsp7KYbUkAv72Ab/7QeT1HMISOBWF7wAUv2aZfw/NV59acnvbzDmDp2l56/Ps2i3mb96W3MKmvhO5a+f1JCwtpeDQYEzIBvqvm7Ibg1f/u7ddXzj9qTLsub09mXJlomY6gfiPGfxuiyD7tGOA0pzXaIWfVaKa3TDmq7Ar55nn+yTXwu5emN1fP38ZV3cKV6Jp6/5tm1tXr+Oqo4WmFnVRD8VLn4yvNvahKFL+ALpAOvEqnKkH16zPaxAr5pLaE4NtWySjpnjpV/zbmefxhm8vzL1PbRR/ja323Lp/nrYypyPH9fRVAJ+Ao9o35UKid5VFBZH0UGEMhO2qGPTlUERBgvkFHsCU3GaoHT+1Zyjy37KEnH5em5PH81AbiN0qnHCgK+RXn+qsia4fkXBHyBWIdvsVP2Uca2M+Nvyz7muQBNsgvNks4Z42h7/tak9Xmaf6udyj7quHmaf5mArx5E1zEqmOYGfO0nMPNVIQFfoWOaRt2U0TH+KuujSPoATKOvPmc9/2JjanuPvs5HGVO7bXnlglU5hGy2T/Y7Ud5qkeZvY6esZj3/MFnny/OPzhtkB3kpzz9Qnn8HqZ7a+4BM2Ud50g3t6S2vw/fNYaBy6+177w/4Fnn+2ohvzwjf5JgFsk8nhd18AV/J8xc6xlXvfhRQIz2LDDaQrenjKjxWSvaxfpDjtcCYR1ehjOmENRK47vE+Ad3zT4OyoedpbK8y/gUFXXwBX1vzP1jz/L2yj3YspfknVT2D1OTWu/D8A8vzd8s+qgxzmDuy2b61+rb2GAIA2Dtp5/mn7XAdLzkP0k5qlj7Iy5Xn7ziGa+4C1/qyAd+qPP8qJ3AXBkzDkQ8/CqisjzLebzqKOZV/7Dz/TgK++mAtp+evBYTz9tdRBcaSx3qCM+ALAPsm2xkj6cJeW08MWuQFq6DsLE3zLxPwtT1/vaqnuuZOAr76qTIB37hzVd9h9MRHzn1dn40y1QFlDLHh+XNa2ycJ+HqzfbRKpobnb37nupSmUyvw/H0ylQR8hb7hqnc/CqjH/8KBTnBo/t6Ab/4PKJVEos9jtcDpfSvZx/6B1j1eG+D2/H0B372NVqlOL6P5100vWHnnyvi3Q32Ql7mvYfzrZGRM6VU97aeKMpijXa3gshXwjZ74AmN737H0/QE4J0LRPX9d81e7+e4ykV7V0y/7tMLQXbtH+7fJlX1sbd/TKYjxFzrGVfhsFGi0wg4Cvrbs48jzL+H52yN0fbKPMgb2mrxJwe1snzTg69L826WuO88QBgFlR/hqhc3yBk858/xVhxi3v5M8fx1Xnj+QThsZsuld2xq/fWeNDi/IToSy3y7sVjrbxxPwtbN92h7PvzDgm7bZtV8mz78iKy3Gfxoz0pp/nUoFfBt2wNcxwXhQwvO3g4FjHtlHHcc2oGkwN3vsVEJJjY8v4Lt3slXquvMkkMjzj0f4qkFebU6MaZHs02iZAd9U8+884GvXudGNoa75JyObc2Sf3A6PssbU8Pxdef4+zV87ljmZi7ldq0TAN2+9b75f8fyFnmm0Rl3z78Tz1/L8u/L8o9eibB/XZC3qHID7h9puZ/P8e/X8M15wzfSCDzhSPVXTigK+Rp5/kE4hmQR8HXP1ettpaf4+2Ud14vp1ZKeuNI+tyzCuOImZ7ROms4p55C+F3knpsk80yUu6U5naPnnrvXn+UxTw7cn4E9GbiGgFEYVEtMRa92EiWkVEDxPRK7Xl58fLVhHRh3o5v5DPqGv+pQK+luzTcBj/ICh+irDT73yppsoA2mY7N+DrG+HrUOL2NlpOuSmD/eRhecH2CN9WGCYdRt4k41G9HTaeVtTasSTVs/tsH3tMARAFfF3VTO1bmZGrAv+xgTRzCnB7/n7VXx/xrXdGpjH35vkb7coe2x/wde8zrJ7//QD+BMBN+kIiWgzgQgCnATgfwJeJqEZENQBfAnABgMUA3hJvK1TAqBp/NaNTqYBvi42JS5rtMCOn1ILiEbOp5x59Hvd0Pom3aZ3DZ1iByOsEzNo+Lc+4i32T3Xn+doaJneff0maysm2JM89fC/gmef5Jqmd32T72CF+VytvQqpkapawLjm1Xz8zk+WvlHUIt5tG150+EIEj3c03movbX98nIVx5HwdcpVGT7e0v1ZOYHAWfK1FIAP2DmSQBriGgVgLPidauY+dF4vx/E2z7QSzt87NjXwJu++vsqDj0SPLF9H4giO/XX3787yfkedtQwfzWxR15lis/+8mFcdvOjAKIfyYFmiIu/dyeAVFuvUfGYAWUk1VOCrvmr4xABh0zU42WeVE/HL1UdZ7wWn4MIu2M9Wj82M7BtXwNHzZ7Ibau6VvMcpvHfsbcBAJgdt1efHtEOpBoB33qAR7fsxT9e82ByLGUA1T3adaCZ2y5COn+AfqZ64Nb8P/KT+5KOvl5LO4iiwU268aw7Ar7b9jWS9xNjtUzA19fJjtUItZrD80ck+7UDRthm7G20nZ69Ph5grEaoB+m4Dv0e2E8uevvU/0V03mqsf1V5/icAuFX7vC5eBgBPWMtf6DoAEV0E4CIAWLBgQVeNCALComNmd7XvdGDRMbPxsmccjdvXbDMegYedZxx7KC549nE4c8ER+JMzT8CyNdtw8HgdTz96NmZP1PGrB5/E6fPmYP32/di0az8A4FnHHYZzn3k0rn/wSYTMOGvh0/AHpzwNv3l4M153xvHY12gjDBkXnH4sHtiwC08/ejaOPnRWcs53vuRk7J5s4R0vOgkHjdVw1klHYvOuSYzVArxi8TG44eHNOGP+HCw94wQcP+cgvHnJPGzf18Apc2dj/Y79OP2Ew/Gul56MJQuPwP9aehqeN/8I7J5sYtPOA3jJorlYdMxsvOiUpwEAXvvc4/Dk7gMIiJI2v2TRUbjz8R3YPdnEy049GgDww4vOxhPb9ydt/PyFZ+Bph0Qdw5+/aCFOmPMkzn/2sVi3fT8O1qqgvvucU3Dn2u04avYEzn3WMbjkvEV485L5+O2qrbhp5Vac96yjccrcQ9BsM+5dtwMvWHhksu//tWR+UgfqyEPGMf+Ig/B35z8Th86q40+fPw9PbN+HNVv34pS5s3HEweM4fs5BWLV5N8brAR7cuBt/9MyjcaDRxgMbd6HRDrHwqEOMe6zr9E8/ejbectZ87NwfdSZnzJ+DFz/9KHzgj5+BZWu24ZxT52b+N/7naxZj3hEHYdXmPZh7aNpJXvKKU3HSUYfgz85egIc27sajW/Zgw879WHzcYRirBTj3WUdj5ZN7cNhBY3j16ccBAF566lz8j5edgsXHH4b/um8Tzn/2sXhg4y6c96xjcKAV4l0vPRnPPuFw/PW5i7Bhx348d94c/N35z8ARh4zjlyueRDtkvOF58zJtfPc5p4AA3P3EDrztDxaiHgQ4fs5BWL1lDw4/aAynn3A4AOC4w2bhz1+0EM12iOefeETy/3jRS0/BZKuNW1ZuxdGHTZR6Au4GKir4RUTXAzjWseqjzPyf8TY3APgAMy+PP38RwK3M/O/x568D+K94v/OZ+Z3x8rcCeCEzvzevDUuWLOHly5eXvihBEAQBIKI7mHmJa12h58/M53VxzvUA5muf58XLkLNcEARBmCKqSvW8CsCFRDRBRCcBWARgGYDbASwiopOIaBxRUPiqitogCIIgeOhJ8yeiNwD4VwBzAVxNRHcz8yuZeQURXYEokNsCcDEzt+N93gvgWgA1AN9g5hU9XYEgCILQMYWa/zAgmr8gCELn5Gn+MsJXEARhBiLGXxAEYQYixl8QBGEGIsZfEARhBjISAV8i2gLg8R4OcRSArX1qTj+RdnWGtKszhrVdwPC2bbq160Rmzg6VxogY/14houW+iPcgkXZ1hrSrM4a1XcDwtm0mtUtkH0EQhBmIGH9BEIQZyEwx/pcNugEepF2dIe3qjGFtFzC8bZsx7ZoRmr8gCIJgMlM8f0EQBEFDjL8gCMIMZFob/2GaLJ6IHiOi+4jobiJSk94cSUTXEdHK+PWIKWrLN4hoMxHdry1ztoUivhDfw3uJ6MwpbtcniGh9fN/uJqJXaes+HLfrYSJ6ZYXtmk9EvyGiB4hoBRG9L14+0HuW066B3jMimkVEy4jonrhdn4yXn0REt8Xn/2Fc1h1x6fcfxstvI6KFU9yuy4lojXa/zoiXT9n/fny+GhHdRUQ/jz9Xe7+YeVr+ISoZvRrAyQDGAdwDYPEA2/MYgKOsZZ8B8KH4/YcA/H9T1JaXAjgTwP1FbQHwKkSzsBGAswHcNsXt+gSiWeLsbRfH3+kEgJPi77pWUbuOA3Bm/P5QAI/E5x/oPctp10DvWXzds+P3YwBui+/DFQAujJd/FcB74vf/A8BX4/cXAvhhRffL167LAbzRsf2U/e/H5/tbAN8D8PP4c6X3azp7/mchniyemRsA1GTxw8RSAN+K338LwOun4qTMfBOAbSXbshTAtzniVgBziOi4KWyXj6UAfsDMk8y8BsAqRN95Fe3ayMx3xu93A3gQ0ZzUA71nOe3yMSX3LL7uPfHHsfiPAbwcwI/i5fb9UvfxRwDOJSqYvb2/7fIxZf/7RDQPwKsB/J/4M6Hi+zWdjf8JyE4Wn/fDqBoG8EsiuoOiyekB4Bhm3hi/3wTgmME0Lbctw3Af3xs/dn9Dk8YG0q74Eft5iLzGoblnVruAAd+zWMK4G8BmANchesrYwcwtx7mTdsXrdwJ42lS0i5nV/bo0vl//TERqZvip/B7/BcAHAYTx56eh4vs1nY3/sPFiZj4TwAUALiail+orOXqGG4q822FqC4CvADgFwBkANgL43KAaQkSzAfwYwCXMvEtfN8h75mjXwO8ZM7eZ+QxE83SfBeCZU90GF3a7iOjZAD6MqH0vAHAkgL+byjYR0WsAbGbmO6byvNPZ+OdNIj/lMPP6+HUzgCsR/SCeVI+R8evmQbUvpy0DvY/M/GT8gw0BfA2pTDGl7SKiMUQG9rvM/JN48cDvmatdw3LP4rbsAPAbAH+ASDZRU8fq507aFa8/HMBTU9Su82P5jJl5EsA3MfX36w8BvI6IHkMkT78cwOdR8f2azsZ/aCaLJ6JDiOhQ9R7AHwO4P27P2+PN3g7gPwfRvhhfW64C8LY48+FsADs1qaNyLI31DYjum2rXhXHmw0kAFgFYVlEbCMDXATzIzP9bWzXQe+Zr16DvGRHNJaI58fuDALwCUTziNwDeGG9m3y91H98I4Nfxk9RUtOshrQMnRLq6fr8q/x6Z+cPMPI+ZFyKyU79m5j9D1fern9HqYftDFK1/BJHe+NEBtuNkRFkW9wBYodqCSKf7FYCVAK4HcOQUtef7iOSAJiIt8b/72oIo0+FL8T28D8CSKW7Xd+Lz3hv/0x+nbf/RuF0PA7igwna9GJGkcy+Au+O/Vw36nuW0a6D3DMBzANwVn/9+AB/TfgfLEAWa/wPARLx8Vvx5Vbz+5Clu16/j+3U/gH9HmhE0Zf/7WhtfhjTbp9L7JeUdBEEQZiDTWfYRBEEQPIjxFwRBmIGI8RcEQZiBiPEXBEGYgYjxFwRBmIGI8RcEQZiBiPEXBEGYgfz/hcuyGWPRrSQAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gyT7tNwkVdS-" + }, + "source": [ + "訓練時間\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "_t-JsKxUViFy", + "outputId": "333aa287-0455-4028-b91c-f83c8d2e1b57" + }, + "source": [ + "print(f\"total time is {end-start} sec\")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "total time is 674.2419369220734 sec\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "u2HaGRVEYGQS" + }, + "source": [ + "## 測試" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "5yFuUKKRYH73", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 500 + }, + "outputId": "7901d4d3-a71b-468e-a12e-6bd9edff551e" + }, + "source": [ + "fix(env, seed)\n", + "agent.network.eval() # 測試前先將 network 切換為 evaluation 模式\n", + "NUM_OF_TEST = 5 # Do not revise it !!!!!\n", + "test_total_reward = []\n", + "action_list = []\n", + "for i in range(NUM_OF_TEST):\n", + " actions = []\n", + " state = env.reset()\n", + "\n", + " img = plt.imshow(env.render(mode='rgb_array'))\n", + "\n", + " total_reward = 0\n", + "\n", + " done = False\n", + " while not done:\n", + " action, _ = agent.sample(state)\n", + " actions.append(action)\n", + " state, reward, done, _ = env.step(action)\n", + "\n", + " total_reward += reward\n", + "\n", + " #img.set_data(env.render(mode='rgb_array'))\n", + " #display.display(plt.gcf())\n", + " #display.clear_output(wait=True)\n", + " print(total_reward)\n", + " test_total_reward.append(total_reward)\n", + "\n", + " action_list.append(actions) #儲存你測試的結果\n", + " print(\"length of actions is \", len(actions))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/torch/__init__.py:422: UserWarning: torch.set_deterministic is deprecated and will be removed in a future release. Please use torch.use_deterministic_algorithms instead\n", + " \"torch.set_deterministic is deprecated and will be removed in a future \"\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "260.62265430635034\n", + "length of actions is 299\n", + "-212.89375915819693\n", + "length of actions is 319\n", + "11.862808485612831\n", + "length of actions is 241\n", + "8.015383611389638\n", + "length of actions is 231\n", + "-219.21903722619058\n", + "length of actions is 256\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAYXUlEQVR4nO3de5BV5Z3u8e9D0zS3RrobbNvuVkR7NFwcYBDxdgZRGSTWwakQhXNGjUNJrGgl1EzlROdUHZ06yVSlMmNmUpnCIeV1ktHxmIuU4wxj1FQm4y2oqAghoiDQIhi5yGVAuvt3/tir2y1N09fN7rf7+VTt2mu9a629fm+7fXrx9rv3UkRgZmbpGFLsAszMrHsc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiSlYcEuaL2mjpE2S7ijUeczMBhsVYh63pBLgt8BVwHbg18CSiFjf5yczMxtkCnXFPQvYFBHvRsQnwKPAwgKdy8xsUBlaoNetBbblrW8HLuxoZ0n++Kb1mZKSUsaUn8bwoWM50vQx+w/uIqKF8tHVDB86ttevf7hpL/sP7KS5+Sjl5dWMHFbF0eaDHDi0iyNHDvZBD8xyIkLHay9UcHdK0jJgWbHObwPX9OmLmH3eLQwrGc2adx/g+RceoLLyTC6/ZDnnjVuIdNz/F7okIvjN757guf/8W3btepv6+mlcNPXLVAyfwGtbf8ivnv8HDh/e34e9MWuvUEMljUB93npd1tYmIlZGxMyImFmgGmwQqqw8g/qaP2BMWT079r/O5i0v0tR0uEBnCzZvfpEd+9YydMgIaqumU1MzuUDnMvtUoYL710CDpLMkDQMWA6sKdC6zNhMnXkztKTM43LSPxt+9SmPjmwU936FDe3l3y/N8eGg9p46aytkTL2XYsJEFPadZQYZKIqJJ0u3AaqAEuD8i3irEucxaVVVNoP60mYwpq2Pz7l/wzjv/SUtLEwDNzUc59MluNu99ttfnOfTJbpqbj2Zrwdatr3DWGWsZV3sep1dOp67u93n33Rd6fR6zjhRsjDsingKeKtTrm+WThjDhzFmcNmYKh5s+pvGj13j//XVt2z/6aDOvvPYIZWXlvT7XkSP7+eijzW3rBw9+xJZtL3J6xXROG30+EydexLZtr3H0aKGGaGywK9ofJ836Wks08+HBjUSsZ9Om/yCi5TPb33+/cP/o27btNd6ve5WoaGbfvg8YPXo8e/Zs6/xAsx4oyAdwul2EpwNaHygpKaWi4gxaWprYu7exbZjkZBk7tpaamsn89re/oLn5k5N6bhuYOpoO6OA2M+unOgpuf8mUmVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJaZXd8CRtAXYDzQDTRExU1Il8M/ABGALcF1E7OldmWZm1qovrrgvj4hpETEzW78DeCYiGoBnsnUzM+sjhRgqWQg8lC0/BFxbgHOYmQ1avQ3uAP5d0iuSlmVt1RGxI1v+AKju5TnMzCxPb+/yfmlENEo6FXha0m/yN0ZEdHQ/ySzolx1vm5mZdazPbhYs6W7gAHALMCcidkiqAX4REed2cqxvFmxmdow+v1mwpFGSyluXgXnAOmAVcFO2203AEz09h5mZtdfjK25JE4GfZqtDgX+KiG9JqgIeA84A3iM3HXB3J6/lK24zs2N0dMXdZ0MlveHgNjNrr8+HSszMrDgc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klptPglnS/pF2S1uW1VUp6WtLb2XNF1i5J35O0SdIbkmYUsngzs8GoK1fcDwLzj2m7A3gmIhqAZ7J1gKuBhuyxDFjRN2WamVmrToM7In4J7D6meSHwULb8EHBtXvvDkfMiMFZSTV8Va2ZmPR/jro6IHdnyB0B1tlwLbMvbb3vW1o6kZZLWSFrTwxrMzAalob19gYgISdGD41YCKwF6cryZ2WDV0yvuna1DINnzrqy9EajP268uazMzsz7S0+BeBdyULd8EPJHXfmM2u2Q2sC9vSMXMzPqAIk48SiHpEWAOMA7YCdwF/Ax4DDgDeA+4LiJ2SxLwfXKzUA4BN0dEp2PYHioxM2svInS89k6D+2RwcJuZtddRcPuTk2ZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klptPglnS/pF2S1uW13S2pUdLa7LEgb9udkjZJ2ijpjwpVuJnZYNWVmwX/N+AA8HBETMna7gYORMRfH7PvJOARYBZwOvBz4PciormTc/iek2Zmx+jxPScj4pfA7i6eZyHwaEQciYjNwCZyIW5mZn2kN2Pct0t6IxtKqcjaaoFteftsz9rakbRM0hpJa3pRg5nZoNPT4F4BnA1MA3YAf9PdF4iIlRExMyJm9rAGM7NBqUfBHRE7I6I5IlqAH/DpcEgjUJ+3a13WZmZmfaRHwS2pJm/1j4HWGSergMWSyiSdBTQAL/euRDMzyze0sx0kPQLMAcZJ2g7cBcyRNA0IYAvwZYCIeEvSY8B6oAm4rbMZJWZm1j2dTgc8KUV4OqCZWTs9ng5oZmb9i4PbzCwxDm4zs8Q4uM3MEuPgNjNLjIPbzCwxDm4zs8Q4uM3MEuPgNjNLjIPbzCwxDm4zs8Q4uM3MEuPgNjNLjIPbzCwxDm4zs8Q4uM3MEuPgNjNLjIPbzCwxnQa3pHpJz0laL+ktSV/L2islPS3p7ey5ImuXpO9J2iTpDUkzCt0JM7PBpCtX3E3An0fEJGA2cJukScAdwDMR0QA8k60DXE3u7u4NwDJgRZ9XbWY2iHUa3BGxIyJezZb3AxuAWmAh8FC220PAtdnyQuDhyHkRGCupps8rNzMbpLo1xi1pAjAdeAmojogd2aYPgOpsuRbYlnfY9qzt2NdaJmmNpDXdrNnMbFDrcnBLGg38GFgeER/nb4uIAKI7J46IlRExMyJmduc4M7PBrkvBLamUXGj/KCJ+kjXvbB0CyZ53Ze2NQH3e4XVZm5mZ9YGuzCoRcB+wISLuydu0CrgpW74JeCKv/cZsdslsYF/ekIqZmfWScqMcJ9hBuhT4D+BNoCVr/gty49yPAWcA7wHXRcTuLOi/D8wHDgE3R8QJx7EldWuYxcxsMIgIHa+90+A+GRzcZmbtdRTc/uSkmVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZonpys2C6yU9J2m9pLckfS1rv1tSo6S12WNB3jF3StokaaOkPypkB8zMBpuu3Cy4BqiJiFcllQOvANcC1wEHIuKvj9l/EvAIMAs4Hfg58HsR0XyCc/iek2Zmx+jxPScjYkdEvJot7wc2ALUnOGQh8GhEHImIzcAmciFuZmZ9oFtj3JImANOBl7Km2yW9Iel+SRVZWy2wLe+w7Zw46M0A+Ku/+jLf/jZMmQKTJsHppxe7opNvzpw5PPjguSxYAJMnw3nnQUlJsauy/mZoV3eUNBr4MbA8Ij6WtAL4v0Bkz38D/Gk3Xm8ZsKx75dpANnXqRGpqYO7c3PqOHbB+fW753/4NNm2CCPjgA2jucOAtbePHj2fWrANMnpxbb2qC55+Ho0dh+3b42c9y7fv2wf79xavTiqtLwS2plFxo/ygifgIQETvztv8AeDJbbQTq8w6vy9o+IyJWAiuz4z3GbW2UjeqdfvqnV92XX54L7eZmWL0a/uu/csH+wx8Wr85Cav0ZlJbCH/5hbjkC/uRPcsvr1sHGjbnlhx+GnTvbv4YNXF2ZVSLgPmBDRNyT116Tt9sfA+uy5VXAYkllks4CGoCX+65kG4xaWnKh3dQEhw7BwYO58B5MWn9xNTfD4cO5n8HBg7mfjQ0uXbnivgS4AXhT0tqs7S+AJZKmkRsq2QJ8GSAi3pL0GLAeaAJuO9GMErN8EbkH5IYG1mbvuNWr4d13c9t27x74YdX6c2hqgmefhU8+gcZGWLUqt/3AgcH3i8s+1WlwR8SvgONNSXnqBMd8C/hWL+qyQejAAfiXf8kNf7S05MZwP/yw2FWdfGvXwg9+AO+9l/s5bN068H9RWfd0+Y+TZoW2dSvcfXexqyi+e+6BNWuKXYX1Zw5uM7M8o0ePpqamhvnz57N69WqO/ZDikSNH2Lp1a5Gqy3Fwm9mgV15ezpgxY/jqV7/KlClTmDdvHkOGDKHlOGNUu3fv5vHHH28X6Dt37uTee+9tt39TUxN79uzp03od3GY26JSUlDBs2DBqa2u5/vrrufrqqzn//PMZNWoUQ4Z8Otkuf7nVqaeeyle+8pV27c3NzXz9619v175r1y4eeOCB47b/8DjzWVtaWjhy5MgJ63dwm9mgIImRI0fyhS98gYsvvphrrrmGsrIyxo0b1yevX1JSQnl5ebv28vJyvvnNb7ZrP3r0KHfddVe79h07drBixQqeeOKJDs/l4DazAa26upqZM2fypS99ialTp9LQ0HDcK+mTrbS0lNra9t8GUltby3333cfrr7/e4bEObjMbcCZOnEhdXR3f+MY3mDBhApMmTSp2SX3KwW1mA8KECRNoaGjg1ltvZcaMGZx55plIx/1W1OQ5uM0sSSUlJVRVVTFnzhwuv/xyFi1aREVFBSWD4OsUHdxmlozW2SCLFy9m6tSpLF26lLKyMsrKyopd2knl4Dazfk0SI0aMYNGiRcyePZuFCxcybtw4hg0bVuzSisbBbWb9UnV1NRdccAE33ngj559/fr+ZDdIfOLjNrOgmT57MmDFjALjiiiuYN28elZWVTG69o4R9hoPbzE6a+vp6RowYAcDcuXOZN28eAJdddhlVVVVt+w3U2SB9xcFtZn2uoqKibQz6sssu44orrgDgmmuu4bTTTgNyHyf30EfPOLjNrMfKysrapt9deOGFXHbZZQAsXryY+vrcHQyHDRs2qP+QWAgObjPrktar46lTp3LBBRcAcPPNN3PWWWcBMGrUqLZxaissB7eZHdfZZ5/d9lHxuro6br/9dgCqqqqorq4uZmmDXqfBLWk48EugLNv/8Yi4K7sR8KNAFfAKcENEfCKpDHgY+APgI+D6iNhSoPrNrIcaGho+88145557LkuXLm1br62tbbuatv6lK1fcR4C5EXFAUinwK0n/CvwZ8N2IeFTSvcBSYEX2vCcizpG0GPg2cH2B6jezY1RUVHDqqad+pu3zn/982/hzq5kzZ7b7djrP5khDV24WHMCBbLU0ewQwF/gfWftDwN3kgnthtgzwOPB9SYpjbxdhZt2WP1uj1a233kpNTU3b+pQpU5g9e/Zn9pHkGRwDSJfGuCWVkBsOOQf4e+AdYG9ENGW7bAdaf3XXAtsAIqJJ0j5ywym/68O6LVElJSUdfq/E888/z6hRo9rdEmow2bJlC/v27WPkyJEsWLCg3QdQFi1a1G74Yvjw4YPii5XsU10K7ohoBqZJGgv8FDivtyeWtAxY1tvXsd6R1KN/Hk+dOpULL7yw28edc8453HDDDd0+bjA65ZRT2j6sYpavW7NKImKvpOeAi4CxkoZmV911QGO2WyNQD2yXNBQ4hdwfKY99rZXASgBJg/cSq49Nnz6dM844o8v7X3fddcyYMaPb56msrGw3jmpmJ0dXZpWMB45moT0CuIrcHxyfAxaRm1lyE9B6g7RV2foL2fZnPb7dsTFjxjB16tQOt3/xi19smzPbFeecc44D1WyA68oVdw3wUDbOPQR4LCKelLQeeFTSN4HXgPuy/e8D/lHSJmA3sLgAdSfnzDPPZOTIkQBcddVVbR8Brqys5NJLLy1maWaWmK7MKnkDmH6c9neBWcdpPwx8sU+qS1BVVRWlpaWUl5ezfPlySktLgdx3NLReCfsv/GbWG/3ik5OSkppJUFJSwvDhw4Hcx3xvueWWtoBesmQJtbW1bV/+7nmxZtbX+kVwT5w4kSuvvPKE+zz11FO8//77HW6PCFpaWvq0riFDhrQF7yWXXMLnPvc5IPcJsyVLlgCf3vfOV9BmdrL0i+AeO3Ys99577wn32bZtG4cOHepw+8GDB/nOd77D4cOHO9xn586dvPDCCx1uHzVqFFdeeWXbFLnly5e3fSdDdXU1Y8eO7aQnZmaF1y+CuytavyLyRB555JETbt+zZw8bNmzocPuIESOYNm2ahzfMrF9LJrj7QkVFBRdffHGxyzAz6xUPzJqZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJabT4JY0XNLLkl6X9Jakv8zaH5S0WdLa7DEta5ek70naJOkNSd2/hbiZmXWoK1/regSYGxEHJJUCv5L0r9m2r0fE48fsfzXQkD0uBFZkz2Zm1gc6veKOnAPZamn2ONENIhcCD2fHvQiMlVTT+1LNzAy6OMYtqUTSWmAX8HREvJRt+lY2HPJdSWVZWy2wLe/w7VmbmZn1gS4Fd0Q0R8Q0oA6YJWkKcCdwHnABUAl8ozsnlrRM0hpJaz788MNulm1mNnh1a1ZJROwFngPmR8SObDjkCPAAMCvbrRHIv0FkXdZ27GutjIiZETFz/PjxPavezGwQ6sqskvGSxmbLI4CrgN+0jlsrd2fda4F12SGrgBuz2SWzgX0RsaMg1ZuZDUJdmVVSAzwkqYRc0D8WEU9KelbSeEDAWuDWbP+ngAXAJuAQcHPfl21mNnh1GtwR8QYw/TjtczvYP4Dbel+amZkdjz85aWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliFBHFrgFJ+4GNxa6jQMYBvyt2EQUwUPsFA7dv7ldazoyI8cfbMPRkV9KBjRExs9hFFIKkNQOxbwO1XzBw++Z+DRweKjEzS4yD28wsMf0luFcWu4ACGqh9G6j9goHbN/drgOgXf5w0M7Ou6y9X3GZm1kVFD25J8yVtlLRJ0h3Frqe7JN0vaZekdXltlZKelvR29lyRtUvS97K+viFpRvEqPzFJ9ZKek7Re0luSvpa1J903ScMlvSzp9axff5m1nyXppaz+f5Y0LGsvy9Y3ZdsnFLP+zkgqkfSapCez9YHSry2S3pS0VtKarC3p92JvFDW4JZUAfw9cDUwClkiaVMyaeuBBYP4xbXcAz0REA/BMtg65fjZkj2XAipNUY080AX8eEZOA2cBt2X+b1Pt2BJgbEb8PTAPmS5oNfBv4bkScA+wBlmb7LwX2ZO3fzfbrz74GbMhbHyj9Arg8IqblTf1L/b3YcxFRtAdwEbA6b/1O4M5i1tTDfkwA1uWtbwRqsuUacvPUAf4BWHK8/fr7A3gCuGog9Q0YCbwKXEjuAxxDs/a29yWwGrgoWx6a7adi195Bf+rIBdhc4ElAA6FfWY1bgHHHtA2Y92J3H8UeKqkFtuWtb8/aUlcdETuy5Q+A6mw5yf5m/4yeDrzEAOhbNpywFtgFPA28A+yNiKZsl/za2/qVbd8HVJ3cirvsb4H/BbRk61UMjH4BBPDvkl6RtCxrS/692FP95ZOTA1ZEhKRkp+5IGg38GFgeER9LatuWat8iohmYJmks8FPgvCKX1GuSrgF2RcQrkuYUu54CuDQiGiWdCjwt6Tf5G1N9L/ZUsa+4G4H6vPW6rC11OyXVAGTPu7L2pPorqZRcaP8oIn6SNQ+IvgFExF7gOXJDCGMltV7I5Nfe1q9s+ynARye51K64BPjvkrYAj5IbLvk70u8XABHRmD3vIvfLdhYD6L3YXcUO7l8DDdlfvocBi4FVRa6pL6wCbsqWbyI3PtzafmP2V+/ZwL68f+r1K8pdWt8HbIiIe/I2Jd03SeOzK20kjSA3br+BXIAvynY7tl+t/V0EPBvZwGl/EhF3RkRdREwg9//RsxHxP0m8XwCSRkkqb10G5gHrSPy92CvFHmQHFgC/JTfO+L+LXU8P6n8E2AEcJTeWtpTcWOEzwNvAz4HKbF+Rm0XzDvAmMLPY9Z+gX5eSG1d8A1ibPRak3jfgfOC1rF/rgP+TtU8EXgY2Af8PKMvah2frm7LtE4vdhy70cQ7w5EDpV9aH17PHW605kfp7sTcPf3LSzCwxxR4qMTOzbnJwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWL+P8eEF+CNkxTNAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Aex7mcKr0J01", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "f1706a79-1fbd-4d61-bdcd-ab257cb152e5" + }, + "source": [ + "print(f\"Your final reward is : %.2f\"%np.mean(test_total_reward))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Your final reward is : -30.32\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "leyebGYRpqsF" + }, + "source": [ + "Action list 的長相" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "hGAH4YWDpp4u", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "c7f5fa21-7b7a-43a8-8478-df76dce7a4ad" + }, + "source": [ + "print(\"Action list looks like \", action_list)\n", + "print(\"Action list's shape looks like \", np.shape(action_list))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Action list looks like [[1, 1, 1, 1, 2, 2, 1, 2, 1, 2, 1, 2, 2, 2, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2, 2, 2, 1, 1, 2, 2, 1, 2, 2, 1, 2, 3, 2, 2, 3, 2, 2, 3, 3, 3, 2, 2, 3, 2, 3, 3, 3, 3, 3, 3, 2, 2, 3, 3, 3, 3, 3, 2, 3, 2, 2, 3, 3, 3, 2, 2, 3, 2, 2, 3, 2, 3, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 1, 1, 2, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 1, 2, 1, 1, 1, 2, 2, 1, 1, 2, 1, 2, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 1, 2, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 2, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 2, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 2, 3, 2, 2, 2, 2, 3, 2, 2, 3, 2, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 3, 2, 0, 2, 2, 3, 2, 0, 3, 2, 2, 2, 3, 2, 2, 3, 2, 3, 2, 3, 2, 2, 2, 3, 2, 3, 2, 2, 0, 1, 2, 2, 2, 0, 1, 2, 2, 1, 2, 1, 2, 1, 1, 2, 2, 0, 2, 1, 2, 2, 1, 1, 1, 2, 2, 2, 1, 2, 2, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2, 1, 2, 2, 0, 1, 0, 2, 2, 2, 2, 3, 3, 2, 3, 2, 3, 2, 3, 3, 2, 3, 2, 3, 3, 3, 2, 3, 2, 3, 3, 3, 2, 2, 3, 3, 3, 2, 2, 3, 2, 3, 2, 3, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 1, 1, 1, 2, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 2, 1, 1, 1, 2, 2, 1, 2, 1, 1, 2, 1, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 3, 2, 2, 0, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 2, 3, 2, 3, 3, 3, 3, 2, 3, 3, 3, 3, 2, 2, 2, 3, 3, 3, 3, 3, 2, 2, 3, 3, 2, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 2], [2, 1, 2, 2, 1, 1, 2, 2, 1, 1, 1, 2, 2, 2, 2, 2, 1, 2, 2, 2, 0, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 3, 3, 3, 3, 2, 3, 3, 2, 2, 2, 3, 3, 3, 3, 2, 2, 3, 3, 3, 3, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 2, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 2, 2, 1, 1, 2, 1, 2, 2, 1, 2, 2, 1, 1, 2, 2, 2, 2, 1, 2, 1, 2, 1, 2, 2, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 3, 3, 3, 3, 3, 3, 2, 3, 2, 3, 3, 3, 3, 3, 3, 3, 2, 2, 3, 2, 3, 3, 3, 3, 3, 2, 3, 3, 3, 2, 2, 2, 2, 3, 3, 3, 2, 2, 3, 3, 2, 3, 3, 2, 2, 2, 2, 2], [1, 1, 1, 1, 2, 1, 2, 2, 1, 1, 2, 2, 1, 2, 2, 1, 2, 1, 2, 2, 2, 1, 2, 1, 2, 2, 2, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 3, 0, 2, 2, 3, 3, 2, 3, 3, 3, 3, 2, 3, 2, 3, 3, 3, 2, 3, 3, 2, 3, 3, 2, 3, 2, 2, 3, 3, 3, 2, 3, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 1, 2, 1, 1, 1, 2, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 2, 1, 1, 2, 1, 1, 2, 2, 1, 1, 1, 2, 1, 1, 2, 2, 1, 2, 1, 1, 2, 1, 2, 2, 2, 1, 1, 2, 2, 1, 2, 2, 2, 2, 1, 1, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 2, 3, 3, 3, 2, 3, 2, 2, 3, 3, 3, 2, 3, 2, 3, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 2], [1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 2, 2, 0, 1, 2, 2, 0, 2, 3, 2, 3, 2, 3, 2, 3, 2, 2, 3, 2, 3, 3, 2, 3, 3, 2, 2, 3, 3, 3, 2, 2, 3, 2, 3, 2, 3, 3, 2, 3, 2, 2, 2, 3, 2, 3, 2, 2, 2, 2, 3, 2, 3, 2, 2, 2, 2, 3, 2, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2, 2, 1, 2, 2, 1, 2, 2, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1, 2, 1, 2, 2, 1, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 1, 2, 2, 2, 2, 2, 3, 3, 2, 3, 3, 2, 3, 3, 3, 2, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 3, 3, 3, 2, 2, 3, 3, 3, 2, 3, 2, 3, 3, 2, 2, 3, 3, 2, 3, 2, 2, 2, 3, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]\n", + "Action list's shape looks like (5,)\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n", + " return array(a, dtype, copy=False, order=order)\n" + ], + "name": "stderr" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "l7sokqEUtrFY" + }, + "source": [ + "Action 的分布\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "WHdAItjj1nxw", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "5129773b-1f4a-4085-d2bf-3bc2abc2598c" + }, + "source": [ + "distribution = {}\n", + "for actions in action_list:\n", + " for action in actions:\n", + " if action not in distribution.keys():\n", + " distribution[action] = 1\n", + " else:\n", + " distribution[action] += 1\n", + "print(distribution)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "{1: 278, 2: 698, 3: 297, 0: 73}\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ricE0schY75M" + }, + "source": [ + "儲存 Model Testing的結果\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "GZsMkGmIY42b", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "8c55c932-4654-4f8c-f6b0-fa52ac3e8b96" + }, + "source": [ + "PATH = \"Action_List_test.npy\" # 可以改成你想取的名字或路徑\n", + "np.save(PATH ,np.array(action_list)) " + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:2: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n", + " \n" + ], + "name": "stderr" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "asK7WfbkaLjt" + }, + "source": [ + "### 你要交到JudgeBoi的檔案94這個\n", + "儲存結果到本地端 (就是你的電腦裡拉 = = )\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "c-CqyhHzaWAL", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 17 + }, + "outputId": "adfba5e6-a107-49aa-9f98-3c0655c5d6c2" + }, + "source": [ + "from google.colab import files\n", + "files.download(PATH)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "\n", + " async function download(id, filename, size) {\n", + " if (!google.colab.kernel.accessAllowed) {\n", + " return;\n", + " }\n", + " const div = document.createElement('div');\n", + " const label = document.createElement('label');\n", + " label.textContent = `Downloading \"${filename}\": `;\n", + " div.appendChild(label);\n", + " const progress = document.createElement('progress');\n", + " progress.max = size;\n", + " div.appendChild(progress);\n", + " document.body.appendChild(div);\n", + "\n", + " const buffers = [];\n", + " let downloaded = 0;\n", + "\n", + " const channel = await google.colab.kernel.comms.open(id);\n", + " // Send a message to notify the kernel that we're ready.\n", + " channel.send({})\n", + "\n", + " for await (const message of channel.messages) {\n", + " // Send a message to notify the kernel that we're ready.\n", + " channel.send({})\n", + " if (message.buffers) {\n", + " for (const buffer of message.buffers) {\n", + " buffers.push(buffer);\n", + " downloaded += buffer.byteLength;\n", + " progress.value = downloaded;\n", + " }\n", + " }\n", + " }\n", + " const blob = new Blob(buffers, {type: 'application/binary'});\n", + " const a = document.createElement('a');\n", + " a.href = window.URL.createObjectURL(blob);\n", + " a.download = filename;\n", + " div.appendChild(a);\n", + " a.click();\n", + " div.remove();\n", + " }\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "download(\"download_5d13b99b-295d-4ab0-814c-b2d0fff26eff\", \"Action_List_test.npy\", 2999)" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "seT4NUmWmAZ1" + }, + "source": [ + "# Server 測試\n", + "到時候下面會是我們Server上測試的環境，可以給大家看一下自己的表現如何" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "U69c-YTxaw6b", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 412 + }, + "outputId": "50015892-29ae-4665-c66f-880aecf7be8f" + }, + "source": [ + "action_list = np.load(PATH,allow_pickle=True) #到時候你上傳的檔案\n", + "seed = 543 #到時候測試的seed 請不要更改\n", + "fix(env, seed)\n", + "\n", + "agent.network.eval() # 測試前先將 network 切換為 evaluation 模式\n", + "\n", + "test_total_reward = []\n", + "for actions in action_list:\n", + " state = env.reset()\n", + " img = plt.imshow(env.render(mode='rgb_array'))\n", + "\n", + " total_reward = 0\n", + "\n", + " done = False\n", + " # while not done:\n", + " done_count = 0\n", + " for action in actions:\n", + " # action, _ = agent1.sample(state)\n", + " state, reward, done, _ = env.step(action)\n", + " done_count += 1\n", + " total_reward += reward\n", + " if done:\n", + " \n", + " break\n", + " # img.set_data(env.render(mode='rgb_array'))\n", + " # display.display(plt.gcf())\n", + " # display.clear_output(wait=True)\n", + " print(f\"Your reward is : %.2f\"%total_reward)\n", + " test_total_reward.append(total_reward)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/torch/__init__.py:422: UserWarning: torch.set_deterministic is deprecated and will be removed in a future release. Please use torch.use_deterministic_algorithms instead\n", + " \"torch.set_deterministic is deprecated and will be removed in a future \"\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Your reward is : 260.62\n", + "Your reward is : -212.89\n", + "Your reward is : 11.86\n", + "Your reward is : 8.02\n", + "Your reward is : -219.22\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAYXUlEQVR4nO3de5BV5Z3u8e9D0zS3RrobbNvuVkR7NFwcYBDxdgZRGSTWwakQhXNGjUNJrGgl1EzlROdUHZ06yVSlMmNmUpnCIeV1ktHxmIuU4wxj1FQm4y2oqAghoiDQIhi5yGVAuvt3/tir2y1N09fN7rf7+VTt2mu9a629fm+7fXrx9rv3UkRgZmbpGFLsAszMrHsc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiSlYcEuaL2mjpE2S7ijUeczMBhsVYh63pBLgt8BVwHbg18CSiFjf5yczMxtkCnXFPQvYFBHvRsQnwKPAwgKdy8xsUBlaoNetBbblrW8HLuxoZ0n++Kb1mZKSUsaUn8bwoWM50vQx+w/uIqKF8tHVDB86ttevf7hpL/sP7KS5+Sjl5dWMHFbF0eaDHDi0iyNHDvZBD8xyIkLHay9UcHdK0jJgWbHObwPX9OmLmH3eLQwrGc2adx/g+RceoLLyTC6/ZDnnjVuIdNz/F7okIvjN757guf/8W3btepv6+mlcNPXLVAyfwGtbf8ivnv8HDh/e34e9MWuvUEMljUB93npd1tYmIlZGxMyImFmgGmwQqqw8g/qaP2BMWT079r/O5i0v0tR0uEBnCzZvfpEd+9YydMgIaqumU1MzuUDnMvtUoYL710CDpLMkDQMWA6sKdC6zNhMnXkztKTM43LSPxt+9SmPjmwU936FDe3l3y/N8eGg9p46aytkTL2XYsJEFPadZQYZKIqJJ0u3AaqAEuD8i3irEucxaVVVNoP60mYwpq2Pz7l/wzjv/SUtLEwDNzUc59MluNu99ttfnOfTJbpqbj2Zrwdatr3DWGWsZV3sep1dOp67u93n33Rd6fR6zjhRsjDsingKeKtTrm+WThjDhzFmcNmYKh5s+pvGj13j//XVt2z/6aDOvvPYIZWXlvT7XkSP7+eijzW3rBw9+xJZtL3J6xXROG30+EydexLZtr3H0aKGGaGywK9ofJ836Wks08+HBjUSsZ9Om/yCi5TPb33+/cP/o27btNd6ve5WoaGbfvg8YPXo8e/Zs6/xAsx4oyAdwul2EpwNaHygpKaWi4gxaWprYu7exbZjkZBk7tpaamsn89re/oLn5k5N6bhuYOpoO6OA2M+unOgpuf8mUmVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJaZXd8CRtAXYDzQDTRExU1Il8M/ABGALcF1E7OldmWZm1qovrrgvj4hpETEzW78DeCYiGoBnsnUzM+sjhRgqWQg8lC0/BFxbgHOYmQ1avQ3uAP5d0iuSlmVt1RGxI1v+AKju5TnMzCxPb+/yfmlENEo6FXha0m/yN0ZEdHQ/ySzolx1vm5mZdazPbhYs6W7gAHALMCcidkiqAX4REed2cqxvFmxmdow+v1mwpFGSyluXgXnAOmAVcFO2203AEz09h5mZtdfjK25JE4GfZqtDgX+KiG9JqgIeA84A3iM3HXB3J6/lK24zs2N0dMXdZ0MlveHgNjNrr8+HSszMrDgc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klptPglnS/pF2S1uW1VUp6WtLb2XNF1i5J35O0SdIbkmYUsngzs8GoK1fcDwLzj2m7A3gmIhqAZ7J1gKuBhuyxDFjRN2WamVmrToM7In4J7D6meSHwULb8EHBtXvvDkfMiMFZSTV8Va2ZmPR/jro6IHdnyB0B1tlwLbMvbb3vW1o6kZZLWSFrTwxrMzAalob19gYgISdGD41YCKwF6cryZ2WDV0yvuna1DINnzrqy9EajP268uazMzsz7S0+BeBdyULd8EPJHXfmM2u2Q2sC9vSMXMzPqAIk48SiHpEWAOMA7YCdwF/Ax4DDgDeA+4LiJ2SxLwfXKzUA4BN0dEp2PYHioxM2svInS89k6D+2RwcJuZtddRcPuTk2ZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klptPglnS/pF2S1uW13S2pUdLa7LEgb9udkjZJ2ijpjwpVuJnZYNWVmwX/N+AA8HBETMna7gYORMRfH7PvJOARYBZwOvBz4PciormTc/iek2Zmx+jxPScj4pfA7i6eZyHwaEQciYjNwCZyIW5mZn2kN2Pct0t6IxtKqcjaaoFteftsz9rakbRM0hpJa3pRg5nZoNPT4F4BnA1MA3YAf9PdF4iIlRExMyJm9rAGM7NBqUfBHRE7I6I5IlqAH/DpcEgjUJ+3a13WZmZmfaRHwS2pJm/1j4HWGSergMWSyiSdBTQAL/euRDMzyze0sx0kPQLMAcZJ2g7cBcyRNA0IYAvwZYCIeEvSY8B6oAm4rbMZJWZm1j2dTgc8KUV4OqCZWTs9ng5oZmb9i4PbzCwxDm4zs8Q4uM3MEuPgNjNLjIPbzCwxDm4zs8Q4uM3MEuPgNjNLjIPbzCwxDm4zs8Q4uM3MEuPgNjNLjIPbzCwxDm4zs8Q4uM3MEuPgNjNLjIPbzCwxnQa3pHpJz0laL+ktSV/L2islPS3p7ey5ImuXpO9J2iTpDUkzCt0JM7PBpCtX3E3An0fEJGA2cJukScAdwDMR0QA8k60DXE3u7u4NwDJgRZ9XbWY2iHUa3BGxIyJezZb3AxuAWmAh8FC220PAtdnyQuDhyHkRGCupps8rNzMbpLo1xi1pAjAdeAmojogd2aYPgOpsuRbYlnfY9qzt2NdaJmmNpDXdrNnMbFDrcnBLGg38GFgeER/nb4uIAKI7J46IlRExMyJmduc4M7PBrkvBLamUXGj/KCJ+kjXvbB0CyZ53Ze2NQH3e4XVZm5mZ9YGuzCoRcB+wISLuydu0CrgpW74JeCKv/cZsdslsYF/ekIqZmfWScqMcJ9hBuhT4D+BNoCVr/gty49yPAWcA7wHXRcTuLOi/D8wHDgE3R8QJx7EldWuYxcxsMIgIHa+90+A+GRzcZmbtdRTc/uSkmVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZonpys2C6yU9J2m9pLckfS1rv1tSo6S12WNB3jF3StokaaOkPypkB8zMBpuu3Cy4BqiJiFcllQOvANcC1wEHIuKvj9l/EvAIMAs4Hfg58HsR0XyCc/iek2Zmx+jxPScjYkdEvJot7wc2ALUnOGQh8GhEHImIzcAmciFuZmZ9oFtj3JImANOBl7Km2yW9Iel+SRVZWy2wLe+w7Zw46M0A+Ku/+jLf/jZMmQKTJsHppxe7opNvzpw5PPjguSxYAJMnw3nnQUlJsauy/mZoV3eUNBr4MbA8Ij6WtAL4v0Bkz38D/Gk3Xm8ZsKx75dpANnXqRGpqYO7c3PqOHbB+fW753/4NNm2CCPjgA2jucOAtbePHj2fWrANMnpxbb2qC55+Ho0dh+3b42c9y7fv2wf79xavTiqtLwS2plFxo/ygifgIQETvztv8AeDJbbQTq8w6vy9o+IyJWAiuz4z3GbW2UjeqdfvqnV92XX54L7eZmWL0a/uu/csH+wx8Wr85Cav0ZlJbCH/5hbjkC/uRPcsvr1sHGjbnlhx+GnTvbv4YNXF2ZVSLgPmBDRNyT116Tt9sfA+uy5VXAYkllks4CGoCX+65kG4xaWnKh3dQEhw7BwYO58B5MWn9xNTfD4cO5n8HBg7mfjQ0uXbnivgS4AXhT0tqs7S+AJZKmkRsq2QJ8GSAi3pL0GLAeaAJuO9GMErN8EbkH5IYG1mbvuNWr4d13c9t27x74YdX6c2hqgmefhU8+gcZGWLUqt/3AgcH3i8s+1WlwR8SvgONNSXnqBMd8C/hWL+qyQejAAfiXf8kNf7S05MZwP/yw2FWdfGvXwg9+AO+9l/s5bN068H9RWfd0+Y+TZoW2dSvcfXexqyi+e+6BNWuKXYX1Zw5uM7M8o0ePpqamhvnz57N69WqO/ZDikSNH2Lp1a5Gqy3Fwm9mgV15ezpgxY/jqV7/KlClTmDdvHkOGDKHlOGNUu3fv5vHHH28X6Dt37uTee+9tt39TUxN79uzp03od3GY26JSUlDBs2DBqa2u5/vrrufrqqzn//PMZNWoUQ4Z8Otkuf7nVqaeeyle+8pV27c3NzXz9619v175r1y4eeOCB47b/8DjzWVtaWjhy5MgJ63dwm9mgIImRI0fyhS98gYsvvphrrrmGsrIyxo0b1yevX1JSQnl5ebv28vJyvvnNb7ZrP3r0KHfddVe79h07drBixQqeeOKJDs/l4DazAa26upqZM2fypS99ialTp9LQ0HDcK+mTrbS0lNra9t8GUltby3333cfrr7/e4bEObjMbcCZOnEhdXR3f+MY3mDBhApMmTSp2SX3KwW1mA8KECRNoaGjg1ltvZcaMGZx55plIx/1W1OQ5uM0sSSUlJVRVVTFnzhwuv/xyFi1aREVFBSWD4OsUHdxmlozW2SCLFy9m6tSpLF26lLKyMsrKyopd2knl4Dazfk0SI0aMYNGiRcyePZuFCxcybtw4hg0bVuzSisbBbWb9UnV1NRdccAE33ngj559/fr+ZDdIfOLjNrOgmT57MmDFjALjiiiuYN28elZWVTG69o4R9hoPbzE6a+vp6RowYAcDcuXOZN28eAJdddhlVVVVt+w3U2SB9xcFtZn2uoqKibQz6sssu44orrgDgmmuu4bTTTgNyHyf30EfPOLjNrMfKysrapt9deOGFXHbZZQAsXryY+vrcHQyHDRs2qP+QWAgObjPrktar46lTp3LBBRcAcPPNN3PWWWcBMGrUqLZxaissB7eZHdfZZ5/d9lHxuro6br/9dgCqqqqorq4uZmmDXqfBLWk48EugLNv/8Yi4K7sR8KNAFfAKcENEfCKpDHgY+APgI+D6iNhSoPrNrIcaGho+88145557LkuXLm1br62tbbuatv6lK1fcR4C5EXFAUinwK0n/CvwZ8N2IeFTSvcBSYEX2vCcizpG0GPg2cH2B6jezY1RUVHDqqad+pu3zn/982/hzq5kzZ7b7djrP5khDV24WHMCBbLU0ewQwF/gfWftDwN3kgnthtgzwOPB9SYpjbxdhZt2WP1uj1a233kpNTU3b+pQpU5g9e/Zn9pHkGRwDSJfGuCWVkBsOOQf4e+AdYG9ENGW7bAdaf3XXAtsAIqJJ0j5ywym/68O6LVElJSUdfq/E888/z6hRo9rdEmow2bJlC/v27WPkyJEsWLCg3QdQFi1a1G74Yvjw4YPii5XsU10K7ohoBqZJGgv8FDivtyeWtAxY1tvXsd6R1KN/Hk+dOpULL7yw28edc8453HDDDd0+bjA65ZRT2j6sYpavW7NKImKvpOeAi4CxkoZmV911QGO2WyNQD2yXNBQ4hdwfKY99rZXASgBJg/cSq49Nnz6dM844o8v7X3fddcyYMaPb56msrGw3jmpmJ0dXZpWMB45moT0CuIrcHxyfAxaRm1lyE9B6g7RV2foL2fZnPb7dsTFjxjB16tQOt3/xi19smzPbFeecc44D1WyA68oVdw3wUDbOPQR4LCKelLQeeFTSN4HXgPuy/e8D/lHSJmA3sLgAdSfnzDPPZOTIkQBcddVVbR8Brqys5NJLLy1maWaWmK7MKnkDmH6c9neBWcdpPwx8sU+qS1BVVRWlpaWUl5ezfPlySktLgdx3NLReCfsv/GbWG/3ik5OSkppJUFJSwvDhw4Hcx3xvueWWtoBesmQJtbW1bV/+7nmxZtbX+kVwT5w4kSuvvPKE+zz11FO8//77HW6PCFpaWvq0riFDhrQF7yWXXMLnPvc5IPcJsyVLlgCf3vfOV9BmdrL0i+AeO3Ys99577wn32bZtG4cOHepw+8GDB/nOd77D4cOHO9xn586dvPDCCx1uHzVqFFdeeWXbFLnly5e3fSdDdXU1Y8eO7aQnZmaF1y+CuytavyLyRB555JETbt+zZw8bNmzocPuIESOYNm2ahzfMrF9LJrj7QkVFBRdffHGxyzAz6xUPzJqZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJabT4JY0XNLLkl6X9Jakv8zaH5S0WdLa7DEta5ek70naJOkNSd2/hbiZmXWoK1/regSYGxEHJJUCv5L0r9m2r0fE48fsfzXQkD0uBFZkz2Zm1gc6veKOnAPZamn2ONENIhcCD2fHvQiMlVTT+1LNzAy6OMYtqUTSWmAX8HREvJRt+lY2HPJdSWVZWy2wLe/w7VmbmZn1gS4Fd0Q0R8Q0oA6YJWkKcCdwHnABUAl8ozsnlrRM0hpJaz788MNulm1mNnh1a1ZJROwFngPmR8SObDjkCPAAMCvbrRHIv0FkXdZ27GutjIiZETFz/PjxPavezGwQ6sqskvGSxmbLI4CrgN+0jlsrd2fda4F12SGrgBuz2SWzgX0RsaMg1ZuZDUJdmVVSAzwkqYRc0D8WEU9KelbSeEDAWuDWbP+ngAXAJuAQcHPfl21mNnh1GtwR8QYw/TjtczvYP4Dbel+amZkdjz85aWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliFBHFrgFJ+4GNxa6jQMYBvyt2EQUwUPsFA7dv7ldazoyI8cfbMPRkV9KBjRExs9hFFIKkNQOxbwO1XzBw++Z+DRweKjEzS4yD28wsMf0luFcWu4ACGqh9G6j9goHbN/drgOgXf5w0M7Ou6y9X3GZm1kVFD25J8yVtlLRJ0h3Frqe7JN0vaZekdXltlZKelvR29lyRtUvS97K+viFpRvEqPzFJ9ZKek7Re0luSvpa1J903ScMlvSzp9axff5m1nyXppaz+f5Y0LGsvy9Y3ZdsnFLP+zkgqkfSapCez9YHSry2S3pS0VtKarC3p92JvFDW4JZUAfw9cDUwClkiaVMyaeuBBYP4xbXcAz0REA/BMtg65fjZkj2XAipNUY080AX8eEZOA2cBt2X+b1Pt2BJgbEb8PTAPmS5oNfBv4bkScA+wBlmb7LwX2ZO3fzfbrz74GbMhbHyj9Arg8IqblTf1L/b3YcxFRtAdwEbA6b/1O4M5i1tTDfkwA1uWtbwRqsuUacvPUAf4BWHK8/fr7A3gCuGog9Q0YCbwKXEjuAxxDs/a29yWwGrgoWx6a7adi195Bf+rIBdhc4ElAA6FfWY1bgHHHtA2Y92J3H8UeKqkFtuWtb8/aUlcdETuy5Q+A6mw5yf5m/4yeDrzEAOhbNpywFtgFPA28A+yNiKZsl/za2/qVbd8HVJ3cirvsb4H/BbRk61UMjH4BBPDvkl6RtCxrS/692FP95ZOTA1ZEhKRkp+5IGg38GFgeER9LatuWat8iohmYJmks8FPgvCKX1GuSrgF2RcQrkuYUu54CuDQiGiWdCjwt6Tf5G1N9L/ZUsa+4G4H6vPW6rC11OyXVAGTPu7L2pPorqZRcaP8oIn6SNQ+IvgFExF7gOXJDCGMltV7I5Nfe1q9s+ynARye51K64BPjvkrYAj5IbLvk70u8XABHRmD3vIvfLdhYD6L3YXcUO7l8DDdlfvocBi4FVRa6pL6wCbsqWbyI3PtzafmP2V+/ZwL68f+r1K8pdWt8HbIiIe/I2Jd03SeOzK20kjSA3br+BXIAvynY7tl+t/V0EPBvZwGl/EhF3RkRdREwg9//RsxHxP0m8XwCSRkkqb10G5gHrSPy92CvFHmQHFgC/JTfO+L+LXU8P6n8E2AEcJTeWtpTcWOEzwNvAz4HKbF+Rm0XzDvAmMLPY9Z+gX5eSG1d8A1ibPRak3jfgfOC1rF/rgP+TtU8EXgY2Af8PKMvah2frm7LtE4vdhy70cQ7w5EDpV9aH17PHW605kfp7sTcPf3LSzCwxxR4qMTOzbnJwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWL+P8eEF+CNkxTNAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TjFBWwQP1hVe" + }, + "source": [ + "# 你的成績" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "GpJpZz3Wbm0X", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "f1b08157-bec6-4c5a-8021-482f719b4ade" + }, + "source": [ + "print(f\"Your final reward is : %.2f\"%np.mean(test_total_reward))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Your final reward is : -30.32\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wUBtYXG2eaqf" + }, + "source": [ + "## 參考資料\n", + "\n", + "以下是一些有用的參考資料。\n", + "建議同學們實做前，可以先參考第一則連結的上課影片。\n", + "在影片的最後有提到兩個有用的 Tips，這對於本次作業的實做非常有幫助。\n", + "\n", + "- [DRL Lecture 1: Policy Gradient (Review)](https://youtu.be/z95ZYgPgXOY)\n", + "- [ML Lecture 23-3: Reinforcement Learning (including Q-learning) start at 30:00](https://youtu.be/2-JNBzCq77c?t=1800)\n", + "- [Lecture 7: Policy Gradient, David Silver](http://www0.cs.ucl.ac.uk/staff/d.silver/web/Teaching_files/pg.pdf)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cGqP2EU1joWM" + }, + "source": [ + "" + ] + } + ] +} \ No newline at end of file diff --git a/11 Quantum ML/作业HW12/hw12_reinforcement_learning_english_version.ipynb b/11 Quantum ML/作业HW12/hw12_reinforcement_learning_english_version.ipynb new file mode 100644 index 0000000..5694e58 --- /dev/null +++ b/11 Quantum ML/作业HW12/hw12_reinforcement_learning_english_version.ipynb @@ -0,0 +1,3092 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "hw12_reinforcement_learning_english_version.ipynb", + "provenance": [], + "collapsed_sections": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "de3a153737af485ea436d7e8393d8248": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_6345f8926212465291c04587353161f1", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_3aa84c8c097d4858a0c38f18dec8b060", + "IPY_MODEL_6647e68cf064416ca593b990bed81edf" + ] + } + }, + "6345f8926212465291c04587353161f1": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "3aa84c8c097d4858a0c38f18dec8b060": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_a25de7edbbee47cc8094f43125efa39b", + "_dom_classes": [], + "description": "Total: 86.3, Final: 0.0: 100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 400, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 400, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_a0bb5061ca0946ab89a89f9abadcebc8" + } + }, + "6647e68cf064416ca593b990bed81edf": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_d99d8e6bcbe0445eb7eddbfe31277635", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 400/400 [11:36<00:00, 1.74s/it]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_5af1c6579f0f4c2eb03df4063749569e" + } + }, + "a25de7edbbee47cc8094f43125efa39b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "a0bb5061ca0946ab89a89f9abadcebc8": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "d99d8e6bcbe0445eb7eddbfe31277635": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "5af1c6579f0f4c2eb03df4063749569e": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + } + } + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Fp30SB4bxeQb" + }, + "source": [ + "# **Homework 12 - Reinforcement Learning**\n", + "\n", + "If you have any problem, e-mail us at ntu-ml-2021spring-ta@googlegroups.com\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yXsnCWPtWSNk" + }, + "source": [ + "## Preliminary work\n", + "\n", + "First, we need to install all necessary packages.\n", + "One of them, gym, builded by OpenAI, is a toolkit for developing Reinforcement Learning algorithm. Other packages are for visualization in colab." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "5e2bScpnkVbv", + "outputId": "52198e39-e2a2-4ea2-a1f3-4ba9545476d7" + }, + "source": [ + "!apt update\n", + "!apt install python-opengl xvfb -y\n", + "!pip install gym[box2d]==0.18.3 pyvirtualdisplay tqdm numpy==1.19.5 torch==1.8.1" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Hit:1 http://security.ubuntu.com/ubuntu bionic-security InRelease\n", + "Ign:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 InRelease\n", + "Hit:3 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease\n", + "Ign:4 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 InRelease\n", + "Hit:5 http://archive.ubuntu.com/ubuntu bionic InRelease\n", + "Hit:6 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 Release\n", + "Hit:7 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease\n", + "Hit:8 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 Release\n", + "Hit:9 http://archive.ubuntu.com/ubuntu bionic-updates InRelease\n", + "Hit:10 http://archive.ubuntu.com/ubuntu bionic-backports InRelease\n", + "Hit:11 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n", + "Hit:12 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease\n", + "Hit:13 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n", + "Reading package lists... Done\n", + "Building dependency tree \n", + "Reading state information... Done\n", + "86 packages can be upgraded. Run 'apt list --upgradable' to see them.\n", + "Reading package lists... Done\n", + "Building dependency tree \n", + "Reading state information... Done\n", + "python-opengl is already the newest version (3.1.0+dfsg-1).\n", + "xvfb is already the newest version (2:1.19.6-1ubuntu4.9).\n", + "The following package was automatically installed and is no longer required:\n", + " libnvidia-common-460\n", + "Use 'apt autoremove' to remove it.\n", + "0 upgraded, 0 newly installed, 0 to remove and 86 not upgraded.\n", + "Requirement already satisfied: gym[box2d] in /usr/local/lib/python3.7/dist-packages (0.17.3)\n", + "Requirement already satisfied: pyvirtualdisplay in /usr/local/lib/python3.7/dist-packages (2.1)\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (4.41.1)\n", + "Requirement already satisfied: numpy>=1.10.4 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.19.5)\n", + "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.4.1)\n", + "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.5.0)\n", + "Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.3.0)\n", + "Requirement already satisfied: box2d-py~=2.3.5; extra == \"box2d\" in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (2.3.8)\n", + "Requirement already satisfied: EasyProcess in /usr/local/lib/python3.7/dist-packages (from pyvirtualdisplay) (0.3)\n", + "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym[box2d]) (0.16.0)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "M_-i3cdoYsks" + }, + "source": [ + "\n", + "Next, set up virtual display，and import all necessaary packages." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "nl2nREINDLiw" + }, + "source": [ + "%%capture\n", + "from pyvirtualdisplay import Display\n", + "virtual_display = Display(visible=0, size=(1400, 900))\n", + "virtual_display.start()\n", + "\n", + "%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from IPython import display\n", + "\n", + "import numpy as np\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.optim as optim\n", + "import torch.nn.functional as F\n", + "from torch.distributions import Categorical\n", + "from tqdm.notebook import tqdm" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CaEJ8BUCpN9P" + }, + "source": [ + "# Warning ! Do not revise random seed !!!\n", + "# Your submission on JudgeBoi will not reproduce your result !!!\n", + "Make your HW result to be reproducible.\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "fV9i8i2YkRbO" + }, + "source": [ + "seed = 543 # Do not change this\n", + "def fix(env, seed):\n", + " env.seed(seed)\n", + " env.action_space.seed(seed)\n", + " torch.manual_seed(seed)\n", + " torch.cuda.manual_seed(seed)\n", + " torch.cuda.manual_seed_all(seed)\n", + " np.random.seed(seed)\n", + " random.seed(seed)\n", + " torch.set_deterministic(True)\n", + " torch.backends.cudnn.benchmark = False\n", + " torch.backends.cudnn.deterministic = True" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "He0XDx6bzjgC" + }, + "source": [ + "Last, call gym and build an [Lunar Lander](https://gym.openai.com/envs/LunarLander-v2/) environment." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "N_4-xJcbBt09" + }, + "source": [ + "%%capture\n", + "import gym\n", + "import random\n", + "env = gym.make('LunarLander-v2')\n", + "fix(env, seed) # fix the environment Do not revise this !!!" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NrkVvTrvWZ5H" + }, + "source": [ + "## What Lunar Lander？\n", + "\n", + "“LunarLander-v2”is to simulate the situation when the craft lands on the surface of the moon.\n", + "\n", + "This task is to enable the craft to land \"safely\" at the pad between the two yellow flags.\n", + "> Landing pad is always at coordinates (0,0).\n", + "> Coordinates are the first two numbers in state vector.\n", + "\n", + "![](https://gym.openai.com/assets/docs/aeloop-138c89d44114492fd02822303e6b4b07213010bb14ca5856d2d49d6b62d88e53.svg)\n", + "\n", + "\"LunarLander-v2\" actually includes \"Agent\" and \"Environment\". \n", + "\n", + "In this homework, we will utilize the function `step()` to control the action of \"Agent\". \n", + "\n", + "Then `step()` will return the observation/state and reward given by the \"Environment\"." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bIbp82sljvAt" + }, + "source": [ + "### Observation / State\n", + "\n", + "First, we can take a look at what an Observation / State looks like." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "rsXZra3N9R5T", + "outputId": "a36868de-bbbc-4de9-815b-0b43fc012c96" + }, + "source": [ + "print(env.observation_space)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Box(-inf, inf, (8,), float32)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ezdfoThbAQ49" + }, + "source": [ + "\n", + "`Box(8,)`means that observation is an 8-dim vector\n", + "### Action\n", + "\n", + "Actions can be taken by looks like" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "p1k4dIrBAaKi", + "outputId": "80c453ee-539f-4e40-c5d8-8e9dc8fffaef" + }, + "source": [ + "print(env.action_space)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Discrete(4)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dejXT6PHBrPn" + }, + "source": [ + "`Discrete(4)` implies that there are four kinds of actions can be taken by agent.\n", + "- 0 implies the agent will not take any actions\n", + "- 2 implies the agent will accelerate downward\n", + "- 1, 3 implies the agent will accelerate left and right\n", + "\n", + "Next, we will try to make the agent interact with the environment. \n", + "Before taking any actions, we recommend to call `reset()` function to reset the environment. Also, this function will return the initial state of the environment." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pi4OmrmZgnWA", + "outputId": "2635bdfb-a4dc-442b-a21a-f57af67edc4b" + }, + "source": [ + "initial_state = env.reset()\n", + "print(initial_state)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "[ 0.00396109 1.4083536 0.40119505 -0.11407257 -0.00458307 -0.09087662\n", + " 0. 0. ]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uBx0mEqqgxJ9" + }, + "source": [ + "Then, we try to get a random action from the agent's action space." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "vxkOEXRKgizt", + "outputId": "de93c740-f01c-464e-f436-a2b59e7dc7e5" + }, + "source": [ + "random_action = env.action_space.sample()\n", + "print(random_action)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mns-bO01g0-J" + }, + "source": [ + "More, we can utilize `step()` to make agent act according to the randomly-selected `random_action`.\n", + "The `step()` function will return four values:\n", + "- observation / state\n", + "- reward\n", + "- done (True/ False)\n", + "- Other information" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "E_WViSxGgIk9" + }, + "source": [ + "observation, reward, done, info = env.step(random_action)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "yK7r126kuCNp", + "outputId": "2f3363d9-5bc3-4ba5-86f2-1c4abc89f179" + }, + "source": [ + "print(done)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "False\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GKdS8vOihxhc" + }, + "source": [ + "### Reward\n", + "\n", + "\n", + "> Landing pad is always at coordinates (0,0). Coordinates are the first two numbers in state vector. Reward for moving from the top of the screen to landing pad and zero speed is about 100..140 points. If lander moves away from landing pad it loses reward back. Episode finishes if the lander crashes or comes to rest, receiving additional -100 or +100 points. Each leg ground contact is +10. Firing main engine is -0.3 points each frame. Solved is 200 points. " + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "vxQNs77hi0_7", + "outputId": "4633d678-be4f-4f52-8f91-1b6681642580" + }, + "source": [ + "print(reward)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "-0.8588900517154912\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Mhqp6D-XgHpe" + }, + "source": [ + "### Random Agent\n", + "In the end, before we start training, we can see whether a random agent can successfully land the moon or not." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 269 + }, + "id": "Y3G0bxoccelv", + "outputId": "11ad28c1-058b-4243-bf35-1fdfbb60be9e" + }, + "source": [ + "env.reset()\n", + "\n", + "img = plt.imshow(env.render(mode='rgb_array'))\n", + "\n", + "done = False\n", + "while not done:\n", + " action = env.action_space.sample()\n", + " observation, reward, done, _ = env.step(action)\n", + "\n", + " img.set_data(env.render(mode='rgb_array'))\n", + " display.display(plt.gcf())\n", + " display.clear_output(wait=True)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAd2ElEQVR4nO3de3RW9b3n8fc39xCu4RLC3SCtRYvcL8taKD0cKWtmsC0qTr3AaCnWarvmTOfomTVHzzmrx1W0doZlh5baKrQVag9eWBZQBI8VKSAochWIIVxyAgECgYDkxnf+eHbCIwnk9iRPdvJ5rfWs7P3bez/7+wvP82Hn9+z9bHN3REQkPBLiXYCIiDSOgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREKmxYLbzKab2T4zyzWzx1pqPyIiHY21xHncZpYI7AemAUeBD4C73X1PzHcmItLBtNQR93gg193z3L0cWA7MbKF9iYh0KEkt9Lz9gSNR80eBCVdb2cx0+aaIyBXc3epqb6ngrpeZzQPmxWv/IiJh1VLBXQAMjJofELTVcPfFwGLQEbeISGO01Bj3B8AwM7vOzFKA2cDKFtqXiEiH0iJH3O5eaWY/AN4EEoHfuvvultiXiEhH0yKnAza6CA2ViIjUcrUPJ3XlpIhIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREKmWfecNLN84BxQBVS6+1gzywT+CAwB8oE73f1088oUEZFqsTji/pq7j3T3scH8Y8A6dx8GrAvmRUQkRlpiqGQmsCSYXgLc3gL7EBHpsJob3A68ZWbbzGxe0Jbl7oXB9DEgq5n7EBGRKM0a4wa+4u4FZtYHWGtmn0QvdHc3M69rwyDo59W1TERErs7c68zVxj+R2ZNAKfBdYIq7F5pZNvDv7v7FeraNTREiIu2Iu1td7U0eKjGzDDPrUj0N/C2wC1gJ3B+sdj/welP3ISIitTX5iNvMcoBXg9kk4CV3/4mZ9QReBgYBh4icDlhcz3PpiFtE5ApXO+KO2VBJcyi4RURqi/lQiYiIxIeCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiFTb3Cb2W/NrMjMdkW1ZZrZWjM7EPzsEbSbmS00s1wz22Fmo1uyeBGRjqghR9wvAtOvaHsMWOfuw4B1wTzAN4BhwWMesCg2ZYqISLV6g9vd/wIUX9E8E1gSTC8Bbo9qX+oRm4DuZpYdq2JFRKTpY9xZ7l4YTB8DsoLp/sCRqPWOBm21mNk8M9tqZlubWIOISIeU1NwncHc3M2/CdouBxQBN2V5EpKNq6hH38eohkOBnUdBeAAyMWm9A0CYiIjHS1OBeCdwfTN8PvB7Vfl9wdslEoCRqSEVERGLA3K89SmFmy4ApQC/gOPAE8BrwMjAIOATc6e7FZmbAc0TOQrkAzHX3esewNVQiIlKbu1td7fUGd2tQcIuI1Ha14NaVkyIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQmZeoPbzH5rZkVmtiuq7UkzKzCz7cFjRtSyx80s18z2mdltLVW4iEhH1ZCbBX8VKAWWuvtNQduTQKm7P3PFusOBZcB4oB/wNvAFd6+qZx+656SIyBWafM9Jd/8LUNzA/cwElrt7mbsfBHKJhLiIiMRIc8a4f2BmO4KhlB5BW3/gSNQ6R4O2WsxsnpltNbOtzahBRKTDaWpwLwKGAiOBQuBnjX0Cd1/s7mPdfWwTaxAR6ZCaFNzuftzdq9z9EvBrLg+HFAADo1YdELSJiEiMNCm4zSw7avabQPUZJyuB2WaWambXAcOALc0rUUREoiXVt4KZLQOmAL3M7CjwBDDFzEYCDuQD3wNw991m9jKwB6gEHq7vjBIREWmcek8HbJUidDqgiEgtTT4dUERE2hYFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjI1BvcZjbQzN4xsz1mttvMfhi0Z5rZWjM7EPzsEbSbmS00s1wz22Fmo1u6EyIiHUlDjrgrgb9z9+HAROBhMxsOPAasc/dhwLpgHuAbRO7uPgyYByyKedUiIh1YvcHt7oXu/mEwfQ7YC/QHZgJLgtWWALcH0zOBpR6xCehuZtkxr1xEpINq1Bi3mQ0BRgGbgSx3LwwWHQOygun+wJGozY4GbVc+1zwz22pmWxtZs4hIh9bg4DazzsAK4EfufjZ6mbs74I3Zsbsvdvex7j62MduJiHR0DQpuM0smEtp/cPdXgubj1UMgwc+ioL0AGBi1+YCgTUREYqAhZ5UY8Btgr7s/G7VoJXB/MH0/8HpU+33B2SUTgZKoIRUREWkmi4xyXGMFs68A7wE7gUtB8z8QGed+GRgEHALudPfiIOifA6YDF4C57n7NcWwza9Qwi4hIR+DuVld7vcHdGhTcIiK1XS24deWkiEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQqYhNwseaGbvmNkeM9ttZj8M2p80swIz2x48ZkRt87iZ5ZrZPjO7rSU7ICLS0TTkZsHZQLa7f2hmXYBtwO3AnUCpuz9zxfrDgWXAeKAf8DbwBXevusY+dM9JEZErNPmek+5e6O4fBtPngL1A/2tsMhNY7u5l7n4QyCUS4iIiEgONGuM2syHAKGBz0PQDM9thZr81sx5BW3/gSNRmR7l20IsA8K//+j1++lO46SYYPhz69Yt3Ra1vypQpvPjiF5kxA268EW64ARIT412VtDVJDV3RzDoDK4AfuftZM1sE/Avgwc+fAf+tEc83D5jXuHKlPfvyl3PIzoapUyPzhYWwZ09kes0ayM0Fdzh2DKquOvAWbr1792b8+FJuvDEyX1kJGzdCRQUcPQqvvRZpLymBc+fiV6fEV4OC28ySiYT2H9z9FQB3Px61/NfAG8FsATAwavMBQdvnuPtiYHGwvca4pYYFo3r9+l0+6v7a1yKhXVUFb74Jn30WCfbf/z5+dbak6t9BcjJMnhyZdod77olM79oF+/ZFppcuhePHaz+HtF8NOavEgN8Ae9392aj27KjVvgnsCqZXArPNLNXMrgOGAVtiV7J0RJcuRUK7shIuXIDz5yPh3ZFU/8dVVQUXL0Z+B+fPR3430rE05Ij7FuBeYKeZbQ/a/gG428xGEhkqyQe+B+Duu83sZWAPUAk8fK0zSkSiuUceEBka2B684t58E/LyIsuKi9t/WFX/HiorYf16KC+HggJYuTKyvLS04/3HJZfVG9zuvgGo65SUVdfY5ifAT5pRl3RApaXw5z9Hhj8uXYqM4Z44Ee+qWt/27fDrX8OhQ5Hfw+HD7f8/KmmcBn84KdLSDh+GJ5+MdxXx9+yzsHVrvKuQtkyXvIuIhIyCW0QkZDRUIhITRkJC5EoZ9yrq+yoJkeZQcIvEwLBhX2Xw4LEAHDu2l5KSQgAuXDjDqVMH41matEMKbpFmMjP69x/Bl/rOpFNyJqd7H+R8eeR0mM/KT3Pq7KcAVFSUkZf3PpeCU0SKiw9TWXkxbnVLeCm4RZopK+uL9On2RbqnDSIxIYX05MyaZRVVF7mYeRqAKi9nUN9xuEeC+/ipPZRVnAfg8OGtFBTsbP3iJZQU3CLN1KVLHzql9iYxIaXWsuTENJITL19k3C11UM10/67jqbpURtH5XRQVHWiVWqV9UHCLNENiYgpDhkwgK+PLDVrf7PK1bF1TI1/EcuZifkuUJu2YTgcUaYYePQbSNb0fyYlp8S5FOhAFt8hVdE1KYnT37jXf99A5LY2BPXuSknT5D9WePQfTNa0/SQkKbmk9GioRqUPXpCS+m5PDdRkZ/LmwkLVFRYzOyaFrejonz53jw7w8SEhh0MAx9MkYHu9ypYPREbdIHbLS0hjUqRPJCQnc3L07iWY1p/FVVUUusBk8eCz9e4wmNalbTPfdtWsWSUk6gperU3CL1OFAaSk7S0o4cO4cv87Lo8KNqi4j8B5jOXz2EhVVVaSndyclqQsJFrt7i6WkZDBu9HcYNfLbJCWlxux5pX3RUEkb0L17d1JTUyktLeX8+fPxLkcCvzkYueLRgb59v8SAfrfSK+MLVI3sxYa/LiYxMZkEi91byCyBnJyJDEj7GwqGVFB0Yh+HDulrAqU2BXecJCUlceONNzJz5ky+9a1vMXToUDZs2MD777/PCy+8QGFhYc2f5hIf1d82kpycRs6QSWR3HYmRQEnpfwAwqP8Yene6IXb7c+fUqYN061XAoG6TGHljIefOnaC4+FDM9iHtg4K7lWVkZHDHHXcwceJE7rjjDjIzL19lN336dG677TYeeeQRFi1axM6dO3n11VcV4HE2bNhkhg/+z6Ql9WBXwcvs3r0aALNEzGI52uicOJFH1077MR9Fv+6jGTJkPGfOFHDpUmUM9yNhp+BuJZmZmXzzm9/kxz/+McOGDSMhoe43vJnRp08fnnjiCcrKyti+fTsLFizgwIED7NypS6JbW58+X+BLOdPp0/lGjpXu4NNDGzh79hidO/dusX3u2b+ajKSzDOs5nS8P/RZnzx4nN/cvLbY/CZ96g9vM0oC/AKnB+v/m7k8ENwJeDvQEtgH3unu5maUCS4ExwCngLnfPb6H627SEhATGjBnD448/ztChQxkxYkSjtk9NTWXChAmsWLGCwsJCXnvtNX73u9/xySefcPr06RaqWqolJaUwbNhkBmZO4JJXUnD6Qw4d2gZAnz7XU+kXOXBqNX07jyIxIfJWSkvqQVJC8z5UPHEil/8ov0hm+nX07TyCG4b+DSdP5nHmzNFm90nah4YccZcBU9291MySgQ1mthr478DP3X25mf0SeABYFPw87e7Xm9ls4KfAXS1Uf5vUv39/7rvvPiZNmsS0adNIS2v+qV3Z2dnMnz+f+fPns2nTJn7xi1+wZcsW8vLyqKrSvZhbQq9eOQzMGkPnlL7sKXqND7b+gYsXSwA4dGgb588Xk5CQxNCht9AlNZ1eaal4ykBI7AxARnIW3dOGAJBgCaQkdvncJe8AZZXnOFX6KceP76tpq6i4yN5P1tK3z5cY0e87DOn9VU584QAfbF2mIRMBGnazYAdKg9nk4OHAVOC/Bu1LgCeJBPfMYBrg34DnzMy8nX+zfHp6OuPHj+fuu+9m2rRpDBky5KrDIU1V/aafNGkSEyZMoLy8nGXLlvH0009z7NgxHYXHUEZGL0aNmMWArhM4eX4fBw9v4MyZgprlVVXlNWFbWLiHvunpjOmZSb5155xFjri7d+9PVtYXAUhOTKdP18iFOobRJ+MmkhJSKas6x2flZ7hwofhz+z9//hRbP1pORlof+mR8idTULiQmpii4BWjgGLeZJRIZDrke+AXwKXDG3atfRUeB/sF0f+AIgLtXmlkJkeGUkzGsu00wM3r16sV3v/tdxowZw4wZM2JydN0QCQkJpKWlMWfOHO666y4OHTrEc889x5IlSygvL6eioqJV6oiVtLQ0Nm7cSEZGRhu5e8xFjp3YTueuXbhYdpa8g/9OWloKUPsbAAHOAu8UFwOXA7i4eD/5+ZGx6ZSUTgwYMBKIvG4GDxpHcnI6AGlpyWRkZFBVVUl+fj4lJSV06tSJM2cOs333y/ToMZBDh7dSUXGhJTssIWKNeZOYWXfgVeB/Ay+6+/VB+0BgtbvfZGa7gOnufjRY9ikwwd1PXvFc84B5weyYZvekFSUkJJCVlcWcOXN49NFH6dOnT8yPrpuioqKCU6dOsXbtWn71q1/x17/+tc2dkVL9e+ratSuzZs2qmf/+979PVlZWPEurxSyRlJR03J3y8tieX5+S0qnmjBT3S5SX1w7lV155hY8//pjExBSqqsp57bXXOHny8tuorf3bSuy5u9XV3qjgBjCzfwQ+A/4e6BscVU8CnnT328zszWD6r2aWBBwDel9rqCQ9Pd0vXmz7dwJJTExk2rRpPPbYYwwZMoTBgwfHu6SrKi4upqioiOeee44//elPFBUVxaWOTp068fWvf52EhATMjEcffZTs7GySk5PJycmpNeYrdXN3Dh06RPX75OLFiyxYsIALFy4H/rZt2zh6VB9gtidNDm4z6w1UuPsZM0sH3iLygeP9wIqoDyd3uPv/M7OHgS+7+/zgw8lvufud19rHzTff7IsWLaqZLy4u5umnn6750O3MmTPs3r27wZ2NtRtuuIEHH3yQW265hdGjR5OSUvefy23Vnj17yM/P56mnnmLTpk1UVsZ+nDQ1NZUxY8ZgZowePZrZs2fXtI8aNapN/EXS3u3fv7/miNzdWbhwIQUFl8flCwoKyM/Pj1N10hTNCe4RRD58TCTy3SYvu/s/m1kOkdMBM4GPgHvcvSw4ffB3wCgiA36z3T3vWvsYO3asb9169Ut7i4qK2LRpU818ZWUlzzzzDGfOnKlpi/WHc926dWPq1KnMmTOHcePGkZ2dXf9GbVxFRQVvv/02W7du5fnnn+fw4cONfo5+/frRrVvkS5VGjhxZE9CdOnVi6tSpCug2LDc3lz179tTML1u2jI8//rhmPi8vj7KysniUJlcRs6GSllBfcF/J3WuN723atOlzL8oXXniBgwcv3137s88+o6Sk5JrPm5iYSO/evZk8eTIPPfQQt956a7sMInenoKCA1atX88orr/Dee+/V+o6Ubt26kZ4e+fBsxIgRzJo1C4ApU6aQk5MDRD5ka4+/n47i0qVLn/sg+PXXX+fUqVMArFmzho0bNzbofSMtp10Fd0NcvHjxc+c37969m9WrV9fMf/bZZyxevJjy8nIgcp70I488wty5c0lNTQ3dcEhTlZWVsX79erZs2fK59hkzZjB8eOT0taSkJFJT9U11HUn1mUl79uxh1apV/PGPf+Tw4cOUlZW1yFCb1K3DBXd9Ll26xMmTJ2uO3JOTk8nMzNSHZSJ1KC4upry8nNWrV7N582YOHDjAu+++q4u/WpiCW0RipqSkhGPHjrF48WLy8vLYsWMHeXnX/ChLmkDBLSIt5uDBgxw5coQFCxZQUlLCjh07OHv2bLzLCj0Ft4i0mo0bN7Jv3z6effZZKisryc3N1dh4Eyi4RaRVVZ/9VVVVxYoVK/joo4/4/e9/T2VlJSdOnIh3eaGg4BaRuKqqqqKsrIzTp0/z4osv8v777/Pee+9RWVlJGK6cjgcFt4i0KefPn6e0tJT9+/fz0ksvsXbtWvLz82udX96RXS24dQccEYmLjIwMMjIyyMrK4tZbb6WgoIDS0lLWr1/PW2+9xbp16zh37ly8y2yTdMQtIm2Ou7Njxw5+9rOfsWrVqporOjuaqx1x63plEWlzzIybb76ZpUuXsnr1aubOnUvv3i13n8+wUXCLSJs2btw4nn/+edasWcNDDz1EZmZmvEuKOwW3iLR5CQkJjB49moULF/Luu+8yf/58OnXqFO+y4kbBLSKhkZSUxE033cTChQv54IMPmDt3bqvdLrAtUXCLSOgkJyczfPhwFi9ezLZt27j33nvp3LlzvMtqNQpuEQmtpKQkhg8fzpIlS9iwYQP33ntvhxgDV3CLSOhFn4WyZs0aXnrpJUaMGEFiYmK8S2sRCm4RaVfGjRvH7Nmz2bJlC0uXLmXEiBHt7sYo9Qa3maWZ2RYz+9jMdpvZPwXtL5rZQTPbHjxGBu1mZgvNLNfMdpjZ6JbuhIhINDMjNTWVu+++m82bN/PLX/6SUaNGkZycHO/SYqIhR9xlwFR3vxkYCUw3s4nBsh+7+8jgsT1o+wYwLHjMAxbVekYRkVZgZqSlpTF37lzeeecdFi1axPjx40N/r9R6q/eI0mA2OXhc6zr5mcDSYLtNQHczC/8t0kUk1Lp168YDDzzAqlWrWLJkCZMmTYp3SU3WoP92zCzRzLYDRcBad98cLPpJMBzyczOrvptsf+BI1OZHgzYRkbjr2bMn99xzD2+88QbLly9n3Lhx8S6pRo8ePZg8eTKTJ0++5umNDfp2QHevAkaaWXfgVTO7CXgcOAakAIuBvwf+uaEFmtk8IkMpDBo0qKGbiYjERGZmJnfddRczZsxg1apVPPXUU+zatSumN0DOyMggJyenVvv111/Pgw8+WKu9V69ejB8/HoCxY8de9Xkb9bWu7n7GzN4Bprv7M0FzmZm9APyPYL4AGBi12YCg7crnWkwk8Bk7dmz8v6JQRDqkLl26cOedd3L77bezYsUKFixYwN69eykvL6+1bmpqap1fdnXdddfxwAMP1Grv27cv06ZNq9VuZpjV+cV/DVJvcJtZb6AiCO10YBrwUzPLdvdCi+z9dmBXsMlK4AdmthyYAJS4e2GTKxQRaWHRZ6F8+9vfZtmyZRw7dqzWegMGDGDWrFm12hMSElr1lMOGHHFnA0vMLJHImPjL7v6Gma0PQt2A7cD8YP1VwAwgF7gAzI192SIisVcd4HPmzIl3KddUb3C7+w5gVB3tU6+yvgMPN780ERGpS7hPZhQR6YAU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMubu8a4BMzsH7It3HS2kF3Ay3kW0gPbaL2i/fVO/wmWwu/eua0FSa1dyFfvcfWy8i2gJZra1PfatvfYL2m/f1K/2Q0MlIiIho+AWEQmZthLci+NdQAtqr31rr/2C9ts39audaBMfToqISMO1lSNuERFpoLgHt5lNN7N9ZpZrZo/Fu57GMrPfmlmRme2Kass0s7VmdiD42SNoNzNbGPR1h5mNjl/l12ZmA83sHTPbY2a7zeyHQXuo+2ZmaWa2xcw+Dvr1T0H7dWa2Oaj/j2aWErSnBvO5wfIh8ay/PmaWaGYfmdkbwXx76Ve+me00s+1mtjVoC/VrsTniGtxmlgj8AvgGMBy428yGx7OmJngRmH5F22PAOncfBqwL5iHSz2HBYx6wqJVqbIpK4O/cfTgwEXg4+LcJe9/KgKnufjMwEphuZhOBnwI/d/frgdPAA8H6DwCng/afB+u1ZT8E9kbNt5d+AXzN3UdGnfoX9tdi07l73B7AJODNqPnHgcfjWVMT+zEE2BU1vw/IDqaziZynDvAr4O661mvrD+B1YFp76hvQCfgQmEDkAo6koL3mdQm8CUwKppOC9SzetV+lPwOIBNhU4A3A2kO/ghrzgV5XtLWb12JjH/EeKukPHImaPxq0hV2WuxcG08eArGA6lP0N/oweBWymHfQtGE7YDhQBa4FPgTPuXhmsEl17Tb+C5SVAz9atuMH+D/A/gUvBfE/aR78AHHjLzLaZ2bygLfSvxaZqK1dOtlvu7mYW2lN3zKwzsAL4kbufNbOaZWHtm7tXASPNrDvwKnBDnEtqNjP7T0CRu28zsynxrqcFfMXdC8ysD7DWzD6JXhjW12JTxfuIuwAYGDU/IGgLu+Nmlg0Q/CwK2kPVXzNLJhLaf3D3V4LmdtE3AHc/A7xDZAihu5lVH8hE117Tr2B5N+BUK5faELcA/8XM8oHlRIZL/i/h7xcA7l4Q/Cwi8p/teNrRa7Gx4h3cHwDDgk++U4DZwMo41xQLK4H7g+n7iYwPV7ffF3zqPREoifpTr02xyKH1b4C97v5s1KJQ983MegdH2phZOpFx+71EAnxWsNqV/aru7yxgvQcDp22Juz/u7gPcfQiR99F6d/8OIe8XgJllmFmX6mngb4FdhPy12CzxHmQHZgD7iYwz/q9419OE+pcBhUAFkbG0B4iMFa4DDgBvA5nBukbkLJpPgZ3A2HjXf41+fYXIuOIOYHvwmBH2vgEjgI+Cfu0C/jFozwG2ALnAn4DUoD0tmM8NlufEuw8N6OMU4I320q+gDx8Hj93VORH212JzHrpyUkQkZOI9VCIiIo2k4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZP4/aOAjA9p4Wi8AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "F5paWqo7tWL2" + }, + "source": [ + "## Policy Gradient\n", + "Now, we can build a simple policy network. The network will return one of action in the action space." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "J8tdmeD-tZew" + }, + "source": [ + "class PolicyGradientNetwork(nn.Module):\n", + "\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.fc1 = nn.Linear(8, 16)\n", + " self.fc2 = nn.Linear(16, 16)\n", + " self.fc3 = nn.Linear(16, 4)\n", + "\n", + " def forward(self, state):\n", + " hid = torch.tanh(self.fc1(state))\n", + " hid = torch.tanh(self.fc2(hid))\n", + " return F.softmax(self.fc3(hid), dim=-1)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ynbqJrhIFTC3" + }, + "source": [ + "Then, we need to build a simple agent. The agent will acts according to the output of the policy network above. There are a few things can be done by agent:\n", + "- `learn()`：update the policy network from log probabilities and rewards.\n", + "- `sample()`：After receiving observation from the environment, utilize policy network to tell which action to take. The return values of this function includes action and log probabilities. " + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "zZo-IxJx286z" + }, + "source": [ + "from torch.optim.lr_scheduler import StepLR\n", + "class PolicyGradientAgent():\n", + " \n", + " def __init__(self, network):\n", + " self.network = network\n", + " self.optimizer = optim.SGD(self.network.parameters(), lr=0.001)\n", + " \n", + " def forward(self, state):\n", + " return self.network(state)\n", + " def learn(self, log_probs, rewards):\n", + " loss = (-log_probs * rewards).sum() # You don't need to revise this to pass simple baseline (but you can)\n", + "\n", + " self.optimizer.zero_grad()\n", + " loss.backward()\n", + " self.optimizer.step()\n", + " \n", + " def sample(self, state):\n", + " action_prob = self.network(torch.FloatTensor(state))\n", + " action_dist = Categorical(action_prob)\n", + " action = action_dist.sample()\n", + " log_prob = action_dist.log_prob(action)\n", + " return action.item(), log_prob" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ehPlnTKyRZf9" + }, + "source": [ + "Lastly, build a network and agent to start training." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "GfJIvML-RYjL" + }, + "source": [ + "network = PolicyGradientNetwork()\n", + "agent = PolicyGradientAgent(network)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ouv23glgf5Qt" + }, + "source": [ + "## Trainin Agent\n", + "\n", + "Now let's start to train our agent.\n", + "Through taking all the interactions between agent and environment as training data, the policy network can learn from all these attempts," + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000, + "referenced_widgets": [ + "de3a153737af485ea436d7e8393d8248", + "6345f8926212465291c04587353161f1", + "3aa84c8c097d4858a0c38f18dec8b060", + "6647e68cf064416ca593b990bed81edf", + "a25de7edbbee47cc8094f43125efa39b", + "a0bb5061ca0946ab89a89f9abadcebc8", + "d99d8e6bcbe0445eb7eddbfe31277635", + "5af1c6579f0f4c2eb03df4063749569e" + ] + }, + "id": "vg5rxBBaf38_", + "outputId": "a1b06e39-99d6-4233-eda0-a3d58e77ffed" + }, + "source": [ + "agent.network.train() # Switch network into training mode \n", + "EPISODE_PER_BATCH = 5 # update the agent every 5 episode\n", + "NUM_BATCH = 400 # totally update the agent for 400 time\n", + "\n", + "avg_total_rewards, avg_final_rewards = [], []\n", + "\n", + "prg_bar = tqdm(range(NUM_BATCH))\n", + "for batch in prg_bar:\n", + "\n", + " log_probs, rewards = [], []\n", + " total_rewards, final_rewards = [], []\n", + "\n", + " # collect trajectory\n", + " for episode in range(EPISODE_PER_BATCH):\n", + " \n", + " state = env.reset()\n", + " total_reward, total_step = 0, 0\n", + " seq_rewards = []\n", + " while True:\n", + "\n", + " action, log_prob = agent.sample(state) # at, log(at|st)\n", + " next_state, reward, done, _ = env.step(action)\n", + "\n", + " log_probs.append(log_prob) # [log(a1|s1), log(a2|s2), ...., log(at|st)]\n", + " # seq_rewards.append(reward)\n", + " state = next_state\n", + " total_reward += reward\n", + " total_step += 1\n", + " rewards.append(reward) # change here\n", + " # ! IMPORTANT !\n", + " # Current reward implementation: immediate reward, given action_list : a1, a2, a3 ......\n", + " # rewards : r1, r2 ,r3 ......\n", + " # medium：change \"rewards\" to accumulative decaying reward, given action_list : a1, a2, a3, ......\n", + " # rewards : r1+0.99*r2+0.99^2*r3+......, r2+0.99*r3+0.99^2*r4+...... , r3+0.99*r4+0.99^2*r5+ ......\n", + " # boss : implement DQN\n", + " if done:\n", + " final_rewards.append(reward)\n", + " total_rewards.append(total_reward)\n", + " \n", + " break\n", + "\n", + " print(f\"rewards looks like \", np.shape(rewards)) \n", + " print(f\"log_probs looks like \", np.shape(log_probs)) \n", + " # record training process\n", + " avg_total_reward = sum(total_rewards) / len(total_rewards)\n", + " avg_final_reward = sum(final_rewards) / len(final_rewards)\n", + " avg_total_rewards.append(avg_total_reward)\n", + " avg_final_rewards.append(avg_final_reward)\n", + " prg_bar.set_description(f\"Total: {avg_total_reward: 4.1f}, Final: {avg_final_reward: 4.1f}\")\n", + "\n", + " # update agent\n", + " # rewards = np.concatenate(rewards, axis=0)\n", + " rewards = (rewards - np.mean(rewards)) / (np.std(rewards) + 1e-9) # normalize the reward \n", + " agent.learn(torch.stack(log_probs), torch.from_numpy(rewards))\n", + " print(\"logs prob looks like \", torch.stack(log_probs).size())\n", + " print(\"torch.from_numpy(rewards) looks like \", torch.from_numpy(rewards).size())" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "de3a153737af485ea436d7e8393d8248", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=400.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "rewards looks like (448,)\n", + "log_probs looks like (448,)\n", + "logs prob looks like torch.Size([448])\n", + "torch.from_numpy(rewards) looks like torch.Size([448])\n", + "rewards looks like (515,)\n", + "log_probs looks like (515,)\n", + "logs prob looks like torch.Size([515])\n", + "torch.from_numpy(rewards) looks like torch.Size([515])\n", + "rewards looks like (392,)\n", + "log_probs looks like (392,)\n", + "logs prob looks like torch.Size([392])\n", + "torch.from_numpy(rewards) looks like torch.Size([392])\n", + "rewards looks like (518,)\n", + "log_probs looks like (518,)\n", + "logs prob looks like torch.Size([518])\n", + "torch.from_numpy(rewards) looks like torch.Size([518])\n", + "rewards looks like (472,)\n", + "log_probs looks like (472,)\n", + "logs prob looks like torch.Size([472])\n", + "torch.from_numpy(rewards) looks like torch.Size([472])\n", + "rewards looks like (530,)\n", + "log_probs looks like (530,)\n", + "logs prob looks like torch.Size([530])\n", + "torch.from_numpy(rewards) looks like torch.Size([530])\n", + "rewards looks like (463,)\n", + "log_probs looks like (463,)\n", + "logs prob looks like torch.Size([463])\n", + "torch.from_numpy(rewards) looks like torch.Size([463])\n", + "rewards looks like (540,)\n", + "log_probs looks like (540,)\n", + "logs prob looks like torch.Size([540])\n", + "torch.from_numpy(rewards) looks like torch.Size([540])\n", + "rewards looks like (513,)\n", + "log_probs looks like (513,)\n", + "logs prob looks like torch.Size([513])\n", + "torch.from_numpy(rewards) looks like torch.Size([513])\n", + "rewards looks like (449,)\n", + "log_probs looks like (449,)\n", + "logs prob looks like torch.Size([449])\n", + "torch.from_numpy(rewards) looks like torch.Size([449])\n", + "rewards looks like (602,)\n", + "log_probs looks like (602,)\n", + "logs prob looks like torch.Size([602])\n", + "torch.from_numpy(rewards) looks like torch.Size([602])\n", + "rewards looks like (542,)\n", + "log_probs looks like (542,)\n", + "logs prob looks like torch.Size([542])\n", + "torch.from_numpy(rewards) looks like torch.Size([542])\n", + "rewards looks like (503,)\n", + "log_probs looks like (503,)\n", + "logs prob looks like torch.Size([503])\n", + "torch.from_numpy(rewards) looks like torch.Size([503])\n", + "rewards looks like (470,)\n", + "log_probs looks like (470,)\n", + "logs prob looks like torch.Size([470])\n", + "torch.from_numpy(rewards) looks like torch.Size([470])\n", + "rewards looks like (518,)\n", + "log_probs looks like (518,)\n", + "logs prob looks like torch.Size([518])\n", + "torch.from_numpy(rewards) looks like torch.Size([518])\n", + "rewards looks like (421,)\n", + "log_probs looks like (421,)\n", + "logs prob looks like torch.Size([421])\n", + "torch.from_numpy(rewards) looks like torch.Size([421])\n", + "rewards looks like (592,)\n", + "log_probs looks like (592,)\n", + "logs prob looks like torch.Size([592])\n", + "torch.from_numpy(rewards) looks like torch.Size([592])\n", + "rewards looks like (520,)\n", + "log_probs looks like (520,)\n", + "logs prob looks like torch.Size([520])\n", + "torch.from_numpy(rewards) looks like torch.Size([520])\n", + "rewards looks like (494,)\n", + "log_probs looks like (494,)\n", + "logs prob looks like torch.Size([494])\n", + "torch.from_numpy(rewards) looks like torch.Size([494])\n", + "rewards looks like (461,)\n", + "log_probs looks like (461,)\n", + "logs prob looks like torch.Size([461])\n", + "torch.from_numpy(rewards) looks like torch.Size([461])\n", + "rewards looks like (572,)\n", + "log_probs looks like (572,)\n", + "logs prob looks like torch.Size([572])\n", + "torch.from_numpy(rewards) looks like torch.Size([572])\n", + "rewards looks like (593,)\n", + "log_probs looks like (593,)\n", + "logs prob looks like torch.Size([593])\n", + "torch.from_numpy(rewards) looks like torch.Size([593])\n", + "rewards looks like (569,)\n", + "log_probs looks like (569,)\n", + "logs prob looks like torch.Size([569])\n", + "torch.from_numpy(rewards) looks like torch.Size([569])\n", + "rewards looks like (546,)\n", + "log_probs looks like (546,)\n", + "logs prob looks like torch.Size([546])\n", + "torch.from_numpy(rewards) looks like torch.Size([546])\n", + "rewards looks like (612,)\n", + "log_probs looks like (612,)\n", + "logs prob looks like torch.Size([612])\n", + "torch.from_numpy(rewards) looks like torch.Size([612])\n", + "rewards looks like (534,)\n", + "log_probs looks like (534,)\n", + "logs prob looks like torch.Size([534])\n", + "torch.from_numpy(rewards) looks like torch.Size([534])\n", + "rewards looks like (513,)\n", + "log_probs looks like (513,)\n", + "logs prob looks like torch.Size([513])\n", + "torch.from_numpy(rewards) looks like torch.Size([513])\n", + "rewards looks like (513,)\n", + "log_probs looks like (513,)\n", + "logs prob looks like torch.Size([513])\n", + "torch.from_numpy(rewards) looks like torch.Size([513])\n", + "rewards looks like (535,)\n", + "log_probs looks like (535,)\n", + "logs prob looks like torch.Size([535])\n", + "torch.from_numpy(rewards) looks like torch.Size([535])\n", + "rewards looks like (533,)\n", + "log_probs looks like (533,)\n", + "logs prob looks like torch.Size([533])\n", + "torch.from_numpy(rewards) looks like torch.Size([533])\n", + "rewards looks like (521,)\n", + "log_probs looks like (521,)\n", + "logs prob looks like torch.Size([521])\n", + "torch.from_numpy(rewards) looks like torch.Size([521])\n", + "rewards looks like (566,)\n", + "log_probs looks like (566,)\n", + "logs prob looks like torch.Size([566])\n", + "torch.from_numpy(rewards) looks like torch.Size([566])\n", + "rewards looks like (586,)\n", + "log_probs looks like (586,)\n", + "logs prob looks like torch.Size([586])\n", + "torch.from_numpy(rewards) looks like torch.Size([586])\n", + "rewards looks like (575,)\n", + "log_probs looks like (575,)\n", + "logs prob looks like torch.Size([575])\n", + "torch.from_numpy(rewards) looks like torch.Size([575])\n", + "rewards looks like (709,)\n", + "log_probs looks like (709,)\n", + "logs prob looks like torch.Size([709])\n", + "torch.from_numpy(rewards) looks like torch.Size([709])\n", + "rewards looks like (486,)\n", + "log_probs looks like (486,)\n", + "logs prob looks like torch.Size([486])\n", + "torch.from_numpy(rewards) looks like torch.Size([486])\n", + "rewards looks like (557,)\n", + "log_probs looks like (557,)\n", + "logs prob looks like torch.Size([557])\n", + "torch.from_numpy(rewards) looks like torch.Size([557])\n", + "rewards looks like (517,)\n", + "log_probs looks like (517,)\n", + "logs prob looks like torch.Size([517])\n", + "torch.from_numpy(rewards) looks like torch.Size([517])\n", + "rewards looks like (550,)\n", + "log_probs looks like (550,)\n", + "logs prob looks like torch.Size([550])\n", + "torch.from_numpy(rewards) looks like torch.Size([550])\n", + "rewards looks like (690,)\n", + "log_probs looks like (690,)\n", + "logs prob looks like torch.Size([690])\n", + "torch.from_numpy(rewards) looks like torch.Size([690])\n", + "rewards looks like (591,)\n", + "log_probs looks like (591,)\n", + "logs prob looks like torch.Size([591])\n", + "torch.from_numpy(rewards) looks like torch.Size([591])\n", + "rewards looks like (689,)\n", + "log_probs looks like (689,)\n", + "logs prob looks like torch.Size([689])\n", + "torch.from_numpy(rewards) looks like torch.Size([689])\n", + "rewards looks like (1059,)\n", + "log_probs looks like (1059,)\n", + "logs prob looks like torch.Size([1059])\n", + "torch.from_numpy(rewards) looks like torch.Size([1059])\n", + "rewards looks like (619,)\n", + "log_probs looks like (619,)\n", + "logs prob looks like torch.Size([619])\n", + "torch.from_numpy(rewards) looks like torch.Size([619])\n", + "rewards looks like (527,)\n", + "log_probs looks like (527,)\n", + "logs prob looks like torch.Size([527])\n", + "torch.from_numpy(rewards) looks like torch.Size([527])\n", + "rewards looks like (514,)\n", + "log_probs looks like (514,)\n", + "logs prob looks like torch.Size([514])\n", + "torch.from_numpy(rewards) looks like torch.Size([514])\n", + "rewards looks like (655,)\n", + "log_probs looks like (655,)\n", + "logs prob looks like torch.Size([655])\n", + "torch.from_numpy(rewards) looks like torch.Size([655])\n", + "rewards looks like (667,)\n", + "log_probs looks like (667,)\n", + "logs prob looks like torch.Size([667])\n", + "torch.from_numpy(rewards) looks like torch.Size([667])\n", + "rewards looks like (712,)\n", + "log_probs looks like (712,)\n", + "logs prob looks like torch.Size([712])\n", + "torch.from_numpy(rewards) looks like torch.Size([712])\n", + "rewards looks like (636,)\n", + "log_probs looks like (636,)\n", + "logs prob looks like torch.Size([636])\n", + "torch.from_numpy(rewards) looks like torch.Size([636])\n", + "rewards looks like (620,)\n", + "log_probs looks like (620,)\n", + "logs prob looks like torch.Size([620])\n", + "torch.from_numpy(rewards) looks like torch.Size([620])\n", + "rewards looks like (543,)\n", + "log_probs looks like (543,)\n", + "logs prob looks like torch.Size([543])\n", + "torch.from_numpy(rewards) looks like torch.Size([543])\n", + "rewards looks like (586,)\n", + "log_probs looks like (586,)\n", + "logs prob looks like torch.Size([586])\n", + "torch.from_numpy(rewards) looks like torch.Size([586])\n", + "rewards looks like (498,)\n", + "log_probs looks like (498,)\n", + "logs prob looks like torch.Size([498])\n", + "torch.from_numpy(rewards) looks like torch.Size([498])\n", + "rewards looks like (586,)\n", + "log_probs looks like (586,)\n", + "logs prob looks like torch.Size([586])\n", + "torch.from_numpy(rewards) looks like torch.Size([586])\n", + "rewards looks like (591,)\n", + "log_probs looks like (591,)\n", + "logs prob looks like torch.Size([591])\n", + "torch.from_numpy(rewards) looks like torch.Size([591])\n", + "rewards looks like (693,)\n", + "log_probs looks like (693,)\n", + "logs prob looks like torch.Size([693])\n", + "torch.from_numpy(rewards) looks like torch.Size([693])\n", + "rewards looks like (648,)\n", + "log_probs looks like (648,)\n", + "logs prob looks like torch.Size([648])\n", + "torch.from_numpy(rewards) looks like torch.Size([648])\n", + "rewards looks like (513,)\n", + "log_probs looks like (513,)\n", + "logs prob looks like torch.Size([513])\n", + "torch.from_numpy(rewards) looks like torch.Size([513])\n", + "rewards looks like (574,)\n", + "log_probs looks like (574,)\n", + "logs prob looks like torch.Size([574])\n", + "torch.from_numpy(rewards) looks like torch.Size([574])\n", + "rewards looks like (718,)\n", + "log_probs looks like (718,)\n", + "logs prob looks like torch.Size([718])\n", + "torch.from_numpy(rewards) looks like torch.Size([718])\n", + "rewards looks like (730,)\n", + "log_probs looks like (730,)\n", + "logs prob looks like torch.Size([730])\n", + "torch.from_numpy(rewards) looks like torch.Size([730])\n", + "rewards looks like (668,)\n", + "log_probs looks like (668,)\n", + "logs prob looks like torch.Size([668])\n", + "torch.from_numpy(rewards) looks like torch.Size([668])\n", + "rewards looks like (754,)\n", + "log_probs looks like (754,)\n", + "logs prob looks like torch.Size([754])\n", + "torch.from_numpy(rewards) looks like torch.Size([754])\n", + "rewards looks like (712,)\n", + "log_probs looks like (712,)\n", + "logs prob looks like torch.Size([712])\n", + "torch.from_numpy(rewards) looks like torch.Size([712])\n", + "rewards looks like (470,)\n", + "log_probs looks like (470,)\n", + "logs prob looks like torch.Size([470])\n", + "torch.from_numpy(rewards) looks like torch.Size([470])\n", + "rewards looks like (665,)\n", + "log_probs looks like (665,)\n", + "logs prob looks like torch.Size([665])\n", + "torch.from_numpy(rewards) looks like torch.Size([665])\n", + "rewards looks like (585,)\n", + "log_probs looks like (585,)\n", + "logs prob looks like torch.Size([585])\n", + "torch.from_numpy(rewards) looks like torch.Size([585])\n", + "rewards looks like (512,)\n", + "log_probs looks like (512,)\n", + "logs prob looks like torch.Size([512])\n", + "torch.from_numpy(rewards) looks like torch.Size([512])\n", + "rewards looks like (702,)\n", + "log_probs looks like (702,)\n", + "logs prob looks like torch.Size([702])\n", + "torch.from_numpy(rewards) looks like torch.Size([702])\n", + "rewards looks like (596,)\n", + "log_probs looks like (596,)\n", + "logs prob looks like torch.Size([596])\n", + "torch.from_numpy(rewards) looks like torch.Size([596])\n", + "rewards looks like (626,)\n", + "log_probs looks like (626,)\n", + "logs prob looks like torch.Size([626])\n", + "torch.from_numpy(rewards) looks like torch.Size([626])\n", + "rewards looks like (566,)\n", + "log_probs looks like (566,)\n", + "logs prob looks like torch.Size([566])\n", + "torch.from_numpy(rewards) looks like torch.Size([566])\n", + "rewards looks like (717,)\n", + "log_probs looks like (717,)\n", + "logs prob looks like torch.Size([717])\n", + "torch.from_numpy(rewards) looks like torch.Size([717])\n", + "rewards looks like (708,)\n", + "log_probs looks like (708,)\n", + "logs prob looks like torch.Size([708])\n", + "torch.from_numpy(rewards) looks like torch.Size([708])\n", + "rewards looks like (565,)\n", + "log_probs looks like (565,)\n", + "logs prob looks like torch.Size([565])\n", + "torch.from_numpy(rewards) looks like torch.Size([565])\n", + "rewards looks like (450,)\n", + "log_probs looks like (450,)\n", + "logs prob looks like torch.Size([450])\n", + "torch.from_numpy(rewards) looks like torch.Size([450])\n", + "rewards looks like (584,)\n", + "log_probs looks like (584,)\n", + "logs prob looks like torch.Size([584])\n", + "torch.from_numpy(rewards) looks like torch.Size([584])\n", + "rewards looks like (670,)\n", + "log_probs looks like (670,)\n", + "logs prob looks like torch.Size([670])\n", + "torch.from_numpy(rewards) looks like torch.Size([670])\n", + "rewards looks like (691,)\n", + "log_probs looks like (691,)\n", + "logs prob looks like torch.Size([691])\n", + "torch.from_numpy(rewards) looks like torch.Size([691])\n", + "rewards looks like (760,)\n", + "log_probs looks like (760,)\n", + "logs prob looks like torch.Size([760])\n", + "torch.from_numpy(rewards) looks like torch.Size([760])\n", + "rewards looks like (752,)\n", + "log_probs looks like (752,)\n", + "logs prob looks like torch.Size([752])\n", + "torch.from_numpy(rewards) looks like torch.Size([752])\n", + "rewards looks like (478,)\n", + "log_probs looks like (478,)\n", + "logs prob looks like torch.Size([478])\n", + "torch.from_numpy(rewards) looks like torch.Size([478])\n", + "rewards looks like (553,)\n", + "log_probs looks like (553,)\n", + "logs prob looks like torch.Size([553])\n", + "torch.from_numpy(rewards) looks like torch.Size([553])\n", + "rewards looks like (1660,)\n", + "log_probs looks like (1660,)\n", + "logs prob looks like torch.Size([1660])\n", + "torch.from_numpy(rewards) looks like torch.Size([1660])\n", + "rewards looks like (751,)\n", + "log_probs looks like (751,)\n", + "logs prob looks like torch.Size([751])\n", + "torch.from_numpy(rewards) looks like torch.Size([751])\n", + "rewards looks like (801,)\n", + "log_probs looks like (801,)\n", + "logs prob looks like torch.Size([801])\n", + "torch.from_numpy(rewards) looks like torch.Size([801])\n", + "rewards looks like (715,)\n", + "log_probs looks like (715,)\n", + "logs prob looks like torch.Size([715])\n", + "torch.from_numpy(rewards) looks like torch.Size([715])\n", + "rewards looks like (708,)\n", + "log_probs looks like (708,)\n", + "logs prob looks like torch.Size([708])\n", + "torch.from_numpy(rewards) looks like torch.Size([708])\n", + "rewards looks like (609,)\n", + "log_probs looks like (609,)\n", + "logs prob looks like torch.Size([609])\n", + "torch.from_numpy(rewards) looks like torch.Size([609])\n", + "rewards looks like (732,)\n", + "log_probs looks like (732,)\n", + "logs prob looks like torch.Size([732])\n", + "torch.from_numpy(rewards) looks like torch.Size([732])\n", + "rewards looks like (603,)\n", + "log_probs looks like (603,)\n", + "logs prob looks like torch.Size([603])\n", + "torch.from_numpy(rewards) looks like torch.Size([603])\n", + "rewards looks like (603,)\n", + "log_probs looks like (603,)\n", + "logs prob looks like torch.Size([603])\n", + "torch.from_numpy(rewards) looks like torch.Size([603])\n", + "rewards looks like (665,)\n", + "log_probs looks like (665,)\n", + "logs prob looks like torch.Size([665])\n", + "torch.from_numpy(rewards) looks like torch.Size([665])\n", + "rewards looks like (658,)\n", + "log_probs looks like (658,)\n", + "logs prob looks like torch.Size([658])\n", + "torch.from_numpy(rewards) looks like torch.Size([658])\n", + "rewards looks like (783,)\n", + "log_probs looks like (783,)\n", + "logs prob looks like torch.Size([783])\n", + "torch.from_numpy(rewards) looks like torch.Size([783])\n", + "rewards looks like (652,)\n", + "log_probs looks like (652,)\n", + "logs prob looks like torch.Size([652])\n", + "torch.from_numpy(rewards) looks like torch.Size([652])\n", + "rewards looks like (892,)\n", + "log_probs looks like (892,)\n", + "logs prob looks like torch.Size([892])\n", + "torch.from_numpy(rewards) looks like torch.Size([892])\n", + "rewards looks like (821,)\n", + "log_probs looks like (821,)\n", + "logs prob looks like torch.Size([821])\n", + "torch.from_numpy(rewards) looks like torch.Size([821])\n", + "rewards looks like (986,)\n", + "log_probs looks like (986,)\n", + "logs prob looks like torch.Size([986])\n", + "torch.from_numpy(rewards) looks like torch.Size([986])\n", + "rewards looks like (916,)\n", + "log_probs looks like (916,)\n", + "logs prob looks like torch.Size([916])\n", + "torch.from_numpy(rewards) looks like torch.Size([916])\n", + "rewards looks like (742,)\n", + "log_probs looks like (742,)\n", + "logs prob looks like torch.Size([742])\n", + "torch.from_numpy(rewards) looks like torch.Size([742])\n", + "rewards looks like (604,)\n", + "log_probs looks like (604,)\n", + "logs prob looks like torch.Size([604])\n", + "torch.from_numpy(rewards) looks like torch.Size([604])\n", + "rewards looks like (818,)\n", + "log_probs looks like (818,)\n", + "logs prob looks like torch.Size([818])\n", + "torch.from_numpy(rewards) looks like torch.Size([818])\n", + "rewards looks like (855,)\n", + "log_probs looks like (855,)\n", + "logs prob looks like torch.Size([855])\n", + "torch.from_numpy(rewards) looks like torch.Size([855])\n", + "rewards looks like (795,)\n", + "log_probs looks like (795,)\n", + "logs prob looks like torch.Size([795])\n", + "torch.from_numpy(rewards) looks like torch.Size([795])\n", + "rewards looks like (868,)\n", + "log_probs looks like (868,)\n", + "logs prob looks like torch.Size([868])\n", + "torch.from_numpy(rewards) looks like torch.Size([868])\n", + "rewards looks like (800,)\n", + "log_probs looks like (800,)\n", + "logs prob looks like torch.Size([800])\n", + "torch.from_numpy(rewards) looks like torch.Size([800])\n", + "rewards looks like (820,)\n", + "log_probs looks like (820,)\n", + "logs prob looks like torch.Size([820])\n", + "torch.from_numpy(rewards) looks like torch.Size([820])\n", + "rewards looks like (760,)\n", + "log_probs looks like (760,)\n", + "logs prob looks like torch.Size([760])\n", + "torch.from_numpy(rewards) looks like torch.Size([760])\n", + "rewards looks like (886,)\n", + "log_probs looks like (886,)\n", + "logs prob looks like torch.Size([886])\n", + "torch.from_numpy(rewards) looks like torch.Size([886])\n", + "rewards looks like (1027,)\n", + "log_probs looks like (1027,)\n", + "logs prob looks like torch.Size([1027])\n", + "torch.from_numpy(rewards) looks like torch.Size([1027])\n", + "rewards looks like (819,)\n", + "log_probs looks like (819,)\n", + "logs prob looks like torch.Size([819])\n", + "torch.from_numpy(rewards) looks like torch.Size([819])\n", + "rewards looks like (934,)\n", + "log_probs looks like (934,)\n", + "logs prob looks like torch.Size([934])\n", + "torch.from_numpy(rewards) looks like torch.Size([934])\n", + "rewards looks like (1648,)\n", + "log_probs looks like (1648,)\n", + "logs prob looks like torch.Size([1648])\n", + "torch.from_numpy(rewards) looks like torch.Size([1648])\n", + "rewards looks like (1057,)\n", + "log_probs looks like (1057,)\n", + "logs prob looks like torch.Size([1057])\n", + "torch.from_numpy(rewards) looks like torch.Size([1057])\n", + "rewards looks like (861,)\n", + "log_probs looks like (861,)\n", + "logs prob looks like torch.Size([861])\n", + "torch.from_numpy(rewards) looks like torch.Size([861])\n", + "rewards looks like (1533,)\n", + "log_probs looks like (1533,)\n", + "logs prob looks like torch.Size([1533])\n", + "torch.from_numpy(rewards) looks like torch.Size([1533])\n", + "rewards looks like (920,)\n", + "log_probs looks like (920,)\n", + "logs prob looks like torch.Size([920])\n", + "torch.from_numpy(rewards) looks like torch.Size([920])\n", + "rewards looks like (905,)\n", + "log_probs looks like (905,)\n", + "logs prob looks like torch.Size([905])\n", + "torch.from_numpy(rewards) looks like torch.Size([905])\n", + "rewards looks like (814,)\n", + "log_probs looks like (814,)\n", + "logs prob looks like torch.Size([814])\n", + "torch.from_numpy(rewards) looks like torch.Size([814])\n", + "rewards looks like (809,)\n", + "log_probs looks like (809,)\n", + "logs prob looks like torch.Size([809])\n", + "torch.from_numpy(rewards) looks like torch.Size([809])\n", + "rewards looks like (873,)\n", + "log_probs looks like (873,)\n", + "logs prob looks like torch.Size([873])\n", + "torch.from_numpy(rewards) looks like torch.Size([873])\n", + "rewards looks like (727,)\n", + "log_probs looks like (727,)\n", + "logs prob looks like torch.Size([727])\n", + "torch.from_numpy(rewards) looks like torch.Size([727])\n", + "rewards looks like (1129,)\n", + "log_probs looks like (1129,)\n", + "logs prob looks like torch.Size([1129])\n", + "torch.from_numpy(rewards) looks like torch.Size([1129])\n", + "rewards looks like (1394,)\n", + "log_probs looks like (1394,)\n", + "logs prob looks like torch.Size([1394])\n", + "torch.from_numpy(rewards) looks like torch.Size([1394])\n", + "rewards looks like (884,)\n", + "log_probs looks like (884,)\n", + "logs prob looks like torch.Size([884])\n", + "torch.from_numpy(rewards) looks like torch.Size([884])\n", + "rewards looks like (1132,)\n", + "log_probs looks like (1132,)\n", + "logs prob looks like torch.Size([1132])\n", + "torch.from_numpy(rewards) looks like torch.Size([1132])\n", + "rewards looks like (1007,)\n", + "log_probs looks like (1007,)\n", + "logs prob looks like torch.Size([1007])\n", + "torch.from_numpy(rewards) looks like torch.Size([1007])\n", + "rewards looks like (711,)\n", + "log_probs looks like (711,)\n", + "logs prob looks like torch.Size([711])\n", + "torch.from_numpy(rewards) looks like torch.Size([711])\n", + "rewards looks like (836,)\n", + "log_probs looks like (836,)\n", + "logs prob looks like torch.Size([836])\n", + "torch.from_numpy(rewards) looks like torch.Size([836])\n", + "rewards looks like (1514,)\n", + "log_probs looks like (1514,)\n", + "logs prob looks like torch.Size([1514])\n", + "torch.from_numpy(rewards) looks like torch.Size([1514])\n", + "rewards looks like (896,)\n", + "log_probs looks like (896,)\n", + "logs prob looks like torch.Size([896])\n", + "torch.from_numpy(rewards) looks like torch.Size([896])\n", + "rewards looks like (912,)\n", + "log_probs looks like (912,)\n", + "logs prob looks like torch.Size([912])\n", + "torch.from_numpy(rewards) looks like torch.Size([912])\n", + "rewards looks like (1478,)\n", + "log_probs looks like (1478,)\n", + "logs prob looks like torch.Size([1478])\n", + "torch.from_numpy(rewards) looks like torch.Size([1478])\n", + "rewards looks like (1279,)\n", + "log_probs looks like (1279,)\n", + "logs prob looks like torch.Size([1279])\n", + "torch.from_numpy(rewards) looks like torch.Size([1279])\n", + "rewards looks like (676,)\n", + "log_probs looks like (676,)\n", + "logs prob looks like torch.Size([676])\n", + "torch.from_numpy(rewards) looks like torch.Size([676])\n", + "rewards looks like (1768,)\n", + "log_probs looks like (1768,)\n", + "logs prob looks like torch.Size([1768])\n", + "torch.from_numpy(rewards) looks like torch.Size([1768])\n", + "rewards looks like (897,)\n", + "log_probs looks like (897,)\n", + "logs prob looks like torch.Size([897])\n", + "torch.from_numpy(rewards) looks like torch.Size([897])\n", + "rewards looks like (1119,)\n", + "log_probs looks like (1119,)\n", + "logs prob looks like torch.Size([1119])\n", + "torch.from_numpy(rewards) looks like torch.Size([1119])\n", + "rewards looks like (943,)\n", + "log_probs looks like (943,)\n", + "logs prob looks like torch.Size([943])\n", + "torch.from_numpy(rewards) looks like torch.Size([943])\n", + "rewards looks like (1255,)\n", + "log_probs looks like (1255,)\n", + "logs prob looks like torch.Size([1255])\n", + "torch.from_numpy(rewards) looks like torch.Size([1255])\n", + "rewards looks like (861,)\n", + "log_probs looks like (861,)\n", + "logs prob looks like torch.Size([861])\n", + "torch.from_numpy(rewards) looks like torch.Size([861])\n", + "rewards looks like (1149,)\n", + "log_probs looks like (1149,)\n", + "logs prob looks like torch.Size([1149])\n", + "torch.from_numpy(rewards) looks like torch.Size([1149])\n", + "rewards looks like (1229,)\n", + "log_probs looks like (1229,)\n", + "logs prob looks like torch.Size([1229])\n", + "torch.from_numpy(rewards) looks like torch.Size([1229])\n", + "rewards looks like (1680,)\n", + "log_probs looks like (1680,)\n", + "logs prob looks like torch.Size([1680])\n", + "torch.from_numpy(rewards) looks like torch.Size([1680])\n", + "rewards looks like (1731,)\n", + "log_probs looks like (1731,)\n", + "logs prob looks like torch.Size([1731])\n", + "torch.from_numpy(rewards) looks like torch.Size([1731])\n", + "rewards looks like (1017,)\n", + "log_probs looks like (1017,)\n", + "logs prob looks like torch.Size([1017])\n", + "torch.from_numpy(rewards) looks like torch.Size([1017])\n", + "rewards looks like (990,)\n", + "log_probs looks like (990,)\n", + "logs prob looks like torch.Size([990])\n", + "torch.from_numpy(rewards) looks like torch.Size([990])\n", + "rewards looks like (1020,)\n", + "log_probs looks like (1020,)\n", + "logs prob looks like torch.Size([1020])\n", + "torch.from_numpy(rewards) looks like torch.Size([1020])\n", + "rewards looks like (1240,)\n", + "log_probs looks like (1240,)\n", + "logs prob looks like torch.Size([1240])\n", + "torch.from_numpy(rewards) looks like torch.Size([1240])\n", + "rewards looks like (774,)\n", + "log_probs looks like (774,)\n", + "logs prob looks like torch.Size([774])\n", + "torch.from_numpy(rewards) looks like torch.Size([774])\n", + "rewards looks like (1069,)\n", + "log_probs looks like (1069,)\n", + "logs prob looks like torch.Size([1069])\n", + "torch.from_numpy(rewards) looks like torch.Size([1069])\n", + "rewards looks like (1355,)\n", + "log_probs looks like (1355,)\n", + "logs prob looks like torch.Size([1355])\n", + "torch.from_numpy(rewards) looks like torch.Size([1355])\n", + "rewards looks like (1556,)\n", + "log_probs looks like (1556,)\n", + "logs prob looks like torch.Size([1556])\n", + "torch.from_numpy(rewards) looks like torch.Size([1556])\n", + "rewards looks like (1840,)\n", + "log_probs looks like (1840,)\n", + "logs prob looks like torch.Size([1840])\n", + "torch.from_numpy(rewards) looks like torch.Size([1840])\n", + "rewards looks like (1352,)\n", + "log_probs looks like (1352,)\n", + "logs prob looks like torch.Size([1352])\n", + "torch.from_numpy(rewards) looks like torch.Size([1352])\n", + "rewards looks like (1617,)\n", + "log_probs looks like (1617,)\n", + "logs prob looks like torch.Size([1617])\n", + "torch.from_numpy(rewards) looks like torch.Size([1617])\n", + "rewards looks like (1637,)\n", + "log_probs looks like (1637,)\n", + "logs prob looks like torch.Size([1637])\n", + "torch.from_numpy(rewards) looks like torch.Size([1637])\n", + "rewards looks like (1606,)\n", + "log_probs looks like (1606,)\n", + "logs prob looks like torch.Size([1606])\n", + "torch.from_numpy(rewards) looks like torch.Size([1606])\n", + "rewards looks like (860,)\n", + "log_probs looks like (860,)\n", + "logs prob looks like torch.Size([860])\n", + "torch.from_numpy(rewards) looks like torch.Size([860])\n", + "rewards looks like (1780,)\n", + "log_probs looks like (1780,)\n", + "logs prob looks like torch.Size([1780])\n", + "torch.from_numpy(rewards) looks like torch.Size([1780])\n", + "rewards looks like (2248,)\n", + "log_probs looks like (2248,)\n", + "logs prob looks like torch.Size([2248])\n", + "torch.from_numpy(rewards) looks like torch.Size([2248])\n", + "rewards looks like (1410,)\n", + "log_probs looks like (1410,)\n", + "logs prob looks like torch.Size([1410])\n", + "torch.from_numpy(rewards) looks like torch.Size([1410])\n", + "rewards looks like (557,)\n", + "log_probs looks like (557,)\n", + "logs prob looks like torch.Size([557])\n", + "torch.from_numpy(rewards) looks like torch.Size([557])\n", + "rewards looks like (719,)\n", + "log_probs looks like (719,)\n", + "logs prob looks like torch.Size([719])\n", + "torch.from_numpy(rewards) looks like torch.Size([719])\n", + "rewards looks like (1919,)\n", + "log_probs looks like (1919,)\n", + "logs prob looks like torch.Size([1919])\n", + "torch.from_numpy(rewards) looks like torch.Size([1919])\n", + "rewards looks like (1250,)\n", + "log_probs looks like (1250,)\n", + "logs prob looks like torch.Size([1250])\n", + "torch.from_numpy(rewards) looks like torch.Size([1250])\n", + "rewards looks like (1054,)\n", + "log_probs looks like (1054,)\n", + "logs prob looks like torch.Size([1054])\n", + "torch.from_numpy(rewards) looks like torch.Size([1054])\n", + "rewards looks like (1276,)\n", + "log_probs looks like (1276,)\n", + "logs prob looks like torch.Size([1276])\n", + "torch.from_numpy(rewards) looks like torch.Size([1276])\n", + "rewards looks like (1040,)\n", + "log_probs looks like (1040,)\n", + "logs prob looks like torch.Size([1040])\n", + "torch.from_numpy(rewards) looks like torch.Size([1040])\n", + "rewards looks like (991,)\n", + "log_probs looks like (991,)\n", + "logs prob looks like torch.Size([991])\n", + "torch.from_numpy(rewards) looks like torch.Size([991])\n", + "rewards looks like (1390,)\n", + "log_probs looks like (1390,)\n", + "logs prob looks like torch.Size([1390])\n", + "torch.from_numpy(rewards) looks like torch.Size([1390])\n", + "rewards looks like (1349,)\n", + "log_probs looks like (1349,)\n", + "logs prob looks like torch.Size([1349])\n", + "torch.from_numpy(rewards) looks like torch.Size([1349])\n", + "rewards looks like (1332,)\n", + "log_probs looks like (1332,)\n", + "logs prob looks like torch.Size([1332])\n", + "torch.from_numpy(rewards) looks like torch.Size([1332])\n", + "rewards looks like (1378,)\n", + "log_probs looks like (1378,)\n", + "logs prob looks like torch.Size([1378])\n", + "torch.from_numpy(rewards) looks like torch.Size([1378])\n", + "rewards looks like (1967,)\n", + "log_probs looks like (1967,)\n", + "logs prob looks like torch.Size([1967])\n", + "torch.from_numpy(rewards) looks like torch.Size([1967])\n", + "rewards looks like (1789,)\n", + "log_probs looks like (1789,)\n", + "logs prob looks like torch.Size([1789])\n", + "torch.from_numpy(rewards) looks like torch.Size([1789])\n", + "rewards looks like (1325,)\n", + "log_probs looks like (1325,)\n", + "logs prob looks like torch.Size([1325])\n", + "torch.from_numpy(rewards) looks like torch.Size([1325])\n", + "rewards looks like (1685,)\n", + "log_probs looks like (1685,)\n", + "logs prob looks like torch.Size([1685])\n", + "torch.from_numpy(rewards) looks like torch.Size([1685])\n", + "rewards looks like (1895,)\n", + "log_probs looks like (1895,)\n", + "logs prob looks like torch.Size([1895])\n", + "torch.from_numpy(rewards) looks like torch.Size([1895])\n", + "rewards looks like (1920,)\n", + "log_probs looks like (1920,)\n", + "logs prob looks like torch.Size([1920])\n", + "torch.from_numpy(rewards) looks like torch.Size([1920])\n", + "rewards looks like (1522,)\n", + "log_probs looks like (1522,)\n", + "logs prob looks like torch.Size([1522])\n", + "torch.from_numpy(rewards) looks like torch.Size([1522])\n", + "rewards looks like (1173,)\n", + "log_probs looks like (1173,)\n", + "logs prob looks like torch.Size([1173])\n", + "torch.from_numpy(rewards) looks like torch.Size([1173])\n", + "rewards looks like (2136,)\n", + "log_probs looks like (2136,)\n", + "logs prob looks like torch.Size([2136])\n", + "torch.from_numpy(rewards) looks like torch.Size([2136])\n", + "rewards looks like (1696,)\n", + "log_probs looks like (1696,)\n", + "logs prob looks like torch.Size([1696])\n", + "torch.from_numpy(rewards) looks like torch.Size([1696])\n", + "rewards looks like (568,)\n", + "log_probs looks like (568,)\n", + "logs prob looks like torch.Size([568])\n", + "torch.from_numpy(rewards) looks like torch.Size([568])\n", + "rewards looks like (1475,)\n", + "log_probs looks like (1475,)\n", + "logs prob looks like torch.Size([1475])\n", + "torch.from_numpy(rewards) looks like torch.Size([1475])\n", + "rewards looks like (2470,)\n", + "log_probs looks like (2470,)\n", + "logs prob looks like torch.Size([2470])\n", + "torch.from_numpy(rewards) looks like torch.Size([2470])\n", + "rewards looks like (3053,)\n", + "log_probs looks like (3053,)\n", + "logs prob looks like torch.Size([3053])\n", + "torch.from_numpy(rewards) looks like torch.Size([3053])\n", + "rewards looks like (915,)\n", + "log_probs looks like (915,)\n", + "logs prob looks like torch.Size([915])\n", + "torch.from_numpy(rewards) looks like torch.Size([915])\n", + "rewards looks like (2049,)\n", + "log_probs looks like (2049,)\n", + "logs prob looks like torch.Size([2049])\n", + "torch.from_numpy(rewards) looks like torch.Size([2049])\n", + "rewards looks like (2068,)\n", + "log_probs looks like (2068,)\n", + "logs prob looks like torch.Size([2068])\n", + "torch.from_numpy(rewards) looks like torch.Size([2068])\n", + "rewards looks like (2528,)\n", + "log_probs looks like (2528,)\n", + "logs prob looks like torch.Size([2528])\n", + "torch.from_numpy(rewards) looks like torch.Size([2528])\n", + "rewards looks like (1839,)\n", + "log_probs looks like (1839,)\n", + "logs prob looks like torch.Size([1839])\n", + "torch.from_numpy(rewards) looks like torch.Size([1839])\n", + "rewards looks like (497,)\n", + "log_probs looks like (497,)\n", + "logs prob looks like torch.Size([497])\n", + "torch.from_numpy(rewards) looks like torch.Size([497])\n", + "rewards looks like (627,)\n", + "log_probs looks like (627,)\n", + "logs prob looks like torch.Size([627])\n", + "torch.from_numpy(rewards) looks like torch.Size([627])\n", + "rewards looks like (2354,)\n", + "log_probs looks like (2354,)\n", + "logs prob looks like torch.Size([2354])\n", + "torch.from_numpy(rewards) looks like torch.Size([2354])\n", + "rewards looks like (2394,)\n", + "log_probs looks like (2394,)\n", + "logs prob looks like torch.Size([2394])\n", + "torch.from_numpy(rewards) looks like torch.Size([2394])\n", + "rewards looks like (743,)\n", + "log_probs looks like (743,)\n", + "logs prob looks like torch.Size([743])\n", + "torch.from_numpy(rewards) looks like torch.Size([743])\n", + "rewards looks like (1572,)\n", + "log_probs looks like (1572,)\n", + "logs prob looks like torch.Size([1572])\n", + "torch.from_numpy(rewards) looks like torch.Size([1572])\n", + "rewards looks like (2575,)\n", + "log_probs looks like (2575,)\n", + "logs prob looks like torch.Size([2575])\n", + "torch.from_numpy(rewards) looks like torch.Size([2575])\n", + "rewards looks like (2226,)\n", + "log_probs looks like (2226,)\n", + "logs prob looks like torch.Size([2226])\n", + "torch.from_numpy(rewards) looks like torch.Size([2226])\n", + "rewards looks like (541,)\n", + "log_probs looks like (541,)\n", + "logs prob looks like torch.Size([541])\n", + "torch.from_numpy(rewards) looks like torch.Size([541])\n", + "rewards looks like (820,)\n", + "log_probs looks like (820,)\n", + "logs prob looks like torch.Size([820])\n", + "torch.from_numpy(rewards) looks like torch.Size([820])\n", + "rewards looks like (2584,)\n", + "log_probs looks like (2584,)\n", + "logs prob looks like torch.Size([2584])\n", + "torch.from_numpy(rewards) looks like torch.Size([2584])\n", + "rewards looks like (1792,)\n", + "log_probs looks like (1792,)\n", + "logs prob looks like torch.Size([1792])\n", + "torch.from_numpy(rewards) looks like torch.Size([1792])\n", + "rewards looks like (1613,)\n", + "log_probs looks like (1613,)\n", + "logs prob looks like torch.Size([1613])\n", + "torch.from_numpy(rewards) looks like torch.Size([1613])\n", + "rewards looks like (4300,)\n", + "log_probs looks like (4300,)\n", + "logs prob looks like torch.Size([4300])\n", + "torch.from_numpy(rewards) looks like torch.Size([4300])\n", + "rewards looks like (1602,)\n", + "log_probs looks like (1602,)\n", + "logs prob looks like torch.Size([1602])\n", + "torch.from_numpy(rewards) looks like torch.Size([1602])\n", + "rewards looks like (3313,)\n", + "log_probs looks like (3313,)\n", + "logs prob looks like torch.Size([3313])\n", + "torch.from_numpy(rewards) looks like torch.Size([3313])\n", + "rewards looks like (1538,)\n", + "log_probs looks like (1538,)\n", + "logs prob looks like torch.Size([1538])\n", + "torch.from_numpy(rewards) looks like torch.Size([1538])\n", + "rewards looks like (1824,)\n", + "log_probs looks like (1824,)\n", + "logs prob looks like torch.Size([1824])\n", + "torch.from_numpy(rewards) looks like torch.Size([1824])\n", + "rewards looks like (1320,)\n", + "log_probs looks like (1320,)\n", + "logs prob looks like torch.Size([1320])\n", + "torch.from_numpy(rewards) looks like torch.Size([1320])\n", + "rewards looks like (2077,)\n", + "log_probs looks like (2077,)\n", + "logs prob looks like torch.Size([2077])\n", + "torch.from_numpy(rewards) looks like torch.Size([2077])\n", + "rewards looks like (1995,)\n", + "log_probs looks like (1995,)\n", + "logs prob looks like torch.Size([1995])\n", + "torch.from_numpy(rewards) looks like torch.Size([1995])\n", + "rewards looks like (1089,)\n", + "log_probs looks like (1089,)\n", + "logs prob looks like torch.Size([1089])\n", + "torch.from_numpy(rewards) looks like torch.Size([1089])\n", + "rewards looks like (1135,)\n", + "log_probs looks like (1135,)\n", + "logs prob looks like torch.Size([1135])\n", + "torch.from_numpy(rewards) looks like torch.Size([1135])\n", + "rewards looks like (1617,)\n", + "log_probs looks like (1617,)\n", + "logs prob looks like torch.Size([1617])\n", + "torch.from_numpy(rewards) looks like torch.Size([1617])\n", + "rewards looks like (942,)\n", + "log_probs looks like (942,)\n", + "logs prob looks like torch.Size([942])\n", + "torch.from_numpy(rewards) looks like torch.Size([942])\n", + "rewards looks like (2006,)\n", + "log_probs looks like (2006,)\n", + "logs prob looks like torch.Size([2006])\n", + "torch.from_numpy(rewards) looks like torch.Size([2006])\n", + "rewards looks like (2204,)\n", + "log_probs looks like (2204,)\n", + "logs prob looks like torch.Size([2204])\n", + "torch.from_numpy(rewards) looks like torch.Size([2204])\n", + "rewards looks like (1060,)\n", + "log_probs looks like (1060,)\n", + "logs prob looks like torch.Size([1060])\n", + "torch.from_numpy(rewards) looks like torch.Size([1060])\n", + "rewards looks like (1994,)\n", + "log_probs looks like (1994,)\n", + "logs prob looks like torch.Size([1994])\n", + "torch.from_numpy(rewards) looks like torch.Size([1994])\n", + "rewards looks like (1118,)\n", + "log_probs looks like (1118,)\n", + "logs prob looks like torch.Size([1118])\n", + "torch.from_numpy(rewards) looks like torch.Size([1118])\n", + "rewards looks like (1298,)\n", + "log_probs looks like (1298,)\n", + "logs prob looks like torch.Size([1298])\n", + "torch.from_numpy(rewards) looks like torch.Size([1298])\n", + "rewards looks like (1377,)\n", + "log_probs looks like (1377,)\n", + "logs prob looks like torch.Size([1377])\n", + "torch.from_numpy(rewards) looks like torch.Size([1377])\n", + "rewards looks like (1902,)\n", + "log_probs looks like (1902,)\n", + "logs prob looks like torch.Size([1902])\n", + "torch.from_numpy(rewards) looks like torch.Size([1902])\n", + "rewards looks like (1982,)\n", + "log_probs looks like (1982,)\n", + "logs prob looks like torch.Size([1982])\n", + "torch.from_numpy(rewards) looks like torch.Size([1982])\n", + "rewards looks like (1625,)\n", + "log_probs looks like (1625,)\n", + "logs prob looks like torch.Size([1625])\n", + "torch.from_numpy(rewards) looks like torch.Size([1625])\n", + "rewards looks like (1947,)\n", + "log_probs looks like (1947,)\n", + "logs prob looks like torch.Size([1947])\n", + "torch.from_numpy(rewards) looks like torch.Size([1947])\n", + "rewards looks like (1589,)\n", + "log_probs looks like (1589,)\n", + "logs prob looks like torch.Size([1589])\n", + "torch.from_numpy(rewards) looks like torch.Size([1589])\n", + "rewards looks like (1625,)\n", + "log_probs looks like (1625,)\n", + "logs prob looks like torch.Size([1625])\n", + "torch.from_numpy(rewards) looks like torch.Size([1625])\n", + "rewards looks like (1492,)\n", + "log_probs looks like (1492,)\n", + "logs prob looks like torch.Size([1492])\n", + "torch.from_numpy(rewards) looks like torch.Size([1492])\n", + "rewards looks like (1347,)\n", + "log_probs looks like (1347,)\n", + "logs prob looks like torch.Size([1347])\n", + "torch.from_numpy(rewards) looks like torch.Size([1347])\n", + "rewards looks like (2110,)\n", + "log_probs looks like (2110,)\n", + "logs prob looks like torch.Size([2110])\n", + "torch.from_numpy(rewards) looks like torch.Size([2110])\n", + "rewards looks like (877,)\n", + "log_probs looks like (877,)\n", + "logs prob looks like torch.Size([877])\n", + "torch.from_numpy(rewards) looks like torch.Size([877])\n", + "rewards looks like (1078,)\n", + "log_probs looks like (1078,)\n", + "logs prob looks like torch.Size([1078])\n", + "torch.from_numpy(rewards) looks like torch.Size([1078])\n", + "rewards looks like (2001,)\n", + "log_probs looks like (2001,)\n", + "logs prob looks like torch.Size([2001])\n", + "torch.from_numpy(rewards) looks like torch.Size([2001])\n", + "rewards looks like (1452,)\n", + "log_probs looks like (1452,)\n", + "logs prob looks like torch.Size([1452])\n", + "torch.from_numpy(rewards) looks like torch.Size([1452])\n", + "rewards looks like (1169,)\n", + "log_probs looks like (1169,)\n", + "logs prob looks like torch.Size([1169])\n", + "torch.from_numpy(rewards) looks like torch.Size([1169])\n", + "rewards looks like (1977,)\n", + "log_probs looks like (1977,)\n", + "logs prob looks like torch.Size([1977])\n", + "torch.from_numpy(rewards) looks like torch.Size([1977])\n", + "rewards looks like (1263,)\n", + "log_probs looks like (1263,)\n", + "logs prob looks like torch.Size([1263])\n", + "torch.from_numpy(rewards) looks like torch.Size([1263])\n", + "rewards looks like (2219,)\n", + "log_probs looks like (2219,)\n", + "logs prob looks like torch.Size([2219])\n", + "torch.from_numpy(rewards) looks like torch.Size([2219])\n", + "rewards looks like (1732,)\n", + "log_probs looks like (1732,)\n", + "logs prob looks like torch.Size([1732])\n", + "torch.from_numpy(rewards) looks like torch.Size([1732])\n", + "rewards looks like (1413,)\n", + "log_probs looks like (1413,)\n", + "logs prob looks like torch.Size([1413])\n", + "torch.from_numpy(rewards) looks like torch.Size([1413])\n", + "rewards looks like (1099,)\n", + "log_probs looks like (1099,)\n", + "logs prob looks like torch.Size([1099])\n", + "torch.from_numpy(rewards) looks like torch.Size([1099])\n", + "rewards looks like (1184,)\n", + "log_probs looks like (1184,)\n", + "logs prob looks like torch.Size([1184])\n", + "torch.from_numpy(rewards) looks like torch.Size([1184])\n", + "rewards looks like (1148,)\n", + "log_probs looks like (1148,)\n", + "logs prob looks like torch.Size([1148])\n", + "torch.from_numpy(rewards) looks like torch.Size([1148])\n", + "rewards looks like (1339,)\n", + "log_probs looks like (1339,)\n", + "logs prob looks like torch.Size([1339])\n", + "torch.from_numpy(rewards) looks like torch.Size([1339])\n", + "rewards looks like (2095,)\n", + "log_probs looks like (2095,)\n", + "logs prob looks like torch.Size([2095])\n", + "torch.from_numpy(rewards) looks like torch.Size([2095])\n", + "rewards looks like (1514,)\n", + "log_probs looks like (1514,)\n", + "logs prob looks like torch.Size([1514])\n", + "torch.from_numpy(rewards) looks like torch.Size([1514])\n", + "rewards looks like (1276,)\n", + "log_probs looks like (1276,)\n", + "logs prob looks like torch.Size([1276])\n", + "torch.from_numpy(rewards) looks like torch.Size([1276])\n", + "rewards looks like (1277,)\n", + "log_probs looks like (1277,)\n", + "logs prob looks like torch.Size([1277])\n", + "torch.from_numpy(rewards) looks like torch.Size([1277])\n", + "rewards looks like (1453,)\n", + "log_probs looks like (1453,)\n", + "logs prob looks like torch.Size([1453])\n", + "torch.from_numpy(rewards) looks like torch.Size([1453])\n", + "rewards looks like (1467,)\n", + "log_probs looks like (1467,)\n", + "logs prob looks like torch.Size([1467])\n", + "torch.from_numpy(rewards) looks like torch.Size([1467])\n", + "rewards looks like (1383,)\n", + "log_probs looks like (1383,)\n", + "logs prob looks like torch.Size([1383])\n", + "torch.from_numpy(rewards) looks like torch.Size([1383])\n", + "rewards looks like (1741,)\n", + "log_probs looks like (1741,)\n", + "logs prob looks like torch.Size([1741])\n", + "torch.from_numpy(rewards) looks like torch.Size([1741])\n", + "rewards looks like (1039,)\n", + "log_probs looks like (1039,)\n", + "logs prob looks like torch.Size([1039])\n", + "torch.from_numpy(rewards) looks like torch.Size([1039])\n", + "rewards looks like (1063,)\n", + "log_probs looks like (1063,)\n", + "logs prob looks like torch.Size([1063])\n", + "torch.from_numpy(rewards) looks like torch.Size([1063])\n", + "rewards looks like (1731,)\n", + "log_probs looks like (1731,)\n", + "logs prob looks like torch.Size([1731])\n", + "torch.from_numpy(rewards) looks like torch.Size([1731])\n", + "rewards looks like (2661,)\n", + "log_probs looks like (2661,)\n", + "logs prob looks like torch.Size([2661])\n", + "torch.from_numpy(rewards) looks like torch.Size([2661])\n", + "rewards looks like (704,)\n", + "log_probs looks like (704,)\n", + "logs prob looks like torch.Size([704])\n", + "torch.from_numpy(rewards) looks like torch.Size([704])\n", + "rewards looks like (1389,)\n", + "log_probs looks like (1389,)\n", + "logs prob looks like torch.Size([1389])\n", + "torch.from_numpy(rewards) looks like torch.Size([1389])\n", + "rewards looks like (2131,)\n", + "log_probs looks like (2131,)\n", + "logs prob looks like torch.Size([2131])\n", + "torch.from_numpy(rewards) looks like torch.Size([2131])\n", + "rewards looks like (1779,)\n", + "log_probs looks like (1779,)\n", + "logs prob looks like torch.Size([1779])\n", + "torch.from_numpy(rewards) looks like torch.Size([1779])\n", + "rewards looks like (1415,)\n", + "log_probs looks like (1415,)\n", + "logs prob looks like torch.Size([1415])\n", + "torch.from_numpy(rewards) looks like torch.Size([1415])\n", + "rewards looks like (2320,)\n", + "log_probs looks like (2320,)\n", + "logs prob looks like torch.Size([2320])\n", + "torch.from_numpy(rewards) looks like torch.Size([2320])\n", + "rewards looks like (1147,)\n", + "log_probs looks like (1147,)\n", + "logs prob looks like torch.Size([1147])\n", + "torch.from_numpy(rewards) looks like torch.Size([1147])\n", + "rewards looks like (1022,)\n", + "log_probs looks like (1022,)\n", + "logs prob looks like torch.Size([1022])\n", + "torch.from_numpy(rewards) looks like torch.Size([1022])\n", + "rewards looks like (2141,)\n", + "log_probs looks like (2141,)\n", + "logs prob looks like torch.Size([2141])\n", + "torch.from_numpy(rewards) looks like torch.Size([2141])\n", + "rewards looks like (1362,)\n", + "log_probs looks like (1362,)\n", + "logs prob looks like torch.Size([1362])\n", + "torch.from_numpy(rewards) looks like torch.Size([1362])\n", + "rewards looks like (1450,)\n", + "log_probs looks like (1450,)\n", + "logs prob looks like torch.Size([1450])\n", + "torch.from_numpy(rewards) looks like torch.Size([1450])\n", + "rewards looks like (1546,)\n", + "log_probs looks like (1546,)\n", + "logs prob looks like torch.Size([1546])\n", + "torch.from_numpy(rewards) looks like torch.Size([1546])\n", + "rewards looks like (1166,)\n", + "log_probs looks like (1166,)\n", + "logs prob looks like torch.Size([1166])\n", + "torch.from_numpy(rewards) looks like torch.Size([1166])\n", + "rewards looks like (1647,)\n", + "log_probs looks like (1647,)\n", + "logs prob looks like torch.Size([1647])\n", + "torch.from_numpy(rewards) looks like torch.Size([1647])\n", + "rewards looks like (1205,)\n", + "log_probs looks like (1205,)\n", + "logs prob looks like torch.Size([1205])\n", + "torch.from_numpy(rewards) looks like torch.Size([1205])\n", + "rewards looks like (2098,)\n", + "log_probs looks like (2098,)\n", + "logs prob looks like torch.Size([2098])\n", + "torch.from_numpy(rewards) looks like torch.Size([2098])\n", + "rewards looks like (1940,)\n", + "log_probs looks like (1940,)\n", + "logs prob looks like torch.Size([1940])\n", + "torch.from_numpy(rewards) looks like torch.Size([1940])\n", + "rewards looks like (2191,)\n", + "log_probs looks like (2191,)\n", + "logs prob looks like torch.Size([2191])\n", + "torch.from_numpy(rewards) looks like torch.Size([2191])\n", + "rewards looks like (2740,)\n", + "log_probs looks like (2740,)\n", + "logs prob looks like torch.Size([2740])\n", + "torch.from_numpy(rewards) looks like torch.Size([2740])\n", + "rewards looks like (587,)\n", + "log_probs looks like (587,)\n", + "logs prob looks like torch.Size([587])\n", + "torch.from_numpy(rewards) looks like torch.Size([587])\n", + "rewards looks like (1063,)\n", + "log_probs looks like (1063,)\n", + "logs prob looks like torch.Size([1063])\n", + "torch.from_numpy(rewards) looks like torch.Size([1063])\n", + "rewards looks like (861,)\n", + "log_probs looks like (861,)\n", + "logs prob looks like torch.Size([861])\n", + "torch.from_numpy(rewards) looks like torch.Size([861])\n", + "rewards looks like (1051,)\n", + "log_probs looks like (1051,)\n", + "logs prob looks like torch.Size([1051])\n", + "torch.from_numpy(rewards) looks like torch.Size([1051])\n", + "rewards looks like (1389,)\n", + "log_probs looks like (1389,)\n", + "logs prob looks like torch.Size([1389])\n", + "torch.from_numpy(rewards) looks like torch.Size([1389])\n", + "rewards looks like (1152,)\n", + "log_probs looks like (1152,)\n", + "logs prob looks like torch.Size([1152])\n", + "torch.from_numpy(rewards) looks like torch.Size([1152])\n", + "rewards looks like (1103,)\n", + "log_probs looks like (1103,)\n", + "logs prob looks like torch.Size([1103])\n", + "torch.from_numpy(rewards) looks like torch.Size([1103])\n", + "rewards looks like (1887,)\n", + "log_probs looks like (1887,)\n", + "logs prob looks like torch.Size([1887])\n", + "torch.from_numpy(rewards) looks like torch.Size([1887])\n", + "rewards looks like (1753,)\n", + "log_probs looks like (1753,)\n", + "logs prob looks like torch.Size([1753])\n", + "torch.from_numpy(rewards) looks like torch.Size([1753])\n", + "rewards looks like (1372,)\n", + "log_probs looks like (1372,)\n", + "logs prob looks like torch.Size([1372])\n", + "torch.from_numpy(rewards) looks like torch.Size([1372])\n", + "rewards looks like (1056,)\n", + "log_probs looks like (1056,)\n", + "logs prob looks like torch.Size([1056])\n", + "torch.from_numpy(rewards) looks like torch.Size([1056])\n", + "rewards looks like (1465,)\n", + "log_probs looks like (1465,)\n", + "logs prob looks like torch.Size([1465])\n", + "torch.from_numpy(rewards) looks like torch.Size([1465])\n", + "rewards looks like (3297,)\n", + "log_probs looks like (3297,)\n", + "logs prob looks like torch.Size([3297])\n", + "torch.from_numpy(rewards) looks like torch.Size([3297])\n", + "rewards looks like (2492,)\n", + "log_probs looks like (2492,)\n", + "logs prob looks like torch.Size([2492])\n", + "torch.from_numpy(rewards) looks like torch.Size([2492])\n", + "rewards looks like (1580,)\n", + "log_probs looks like (1580,)\n", + "logs prob looks like torch.Size([1580])\n", + "torch.from_numpy(rewards) looks like torch.Size([1580])\n", + "rewards looks like (1357,)\n", + "log_probs looks like (1357,)\n", + "logs prob looks like torch.Size([1357])\n", + "torch.from_numpy(rewards) looks like torch.Size([1357])\n", + "rewards looks like (1227,)\n", + "log_probs looks like (1227,)\n", + "logs prob looks like torch.Size([1227])\n", + "torch.from_numpy(rewards) looks like torch.Size([1227])\n", + "rewards looks like (2123,)\n", + "log_probs looks like (2123,)\n", + "logs prob looks like torch.Size([2123])\n", + "torch.from_numpy(rewards) looks like torch.Size([2123])\n", + "rewards looks like (1864,)\n", + "log_probs looks like (1864,)\n", + "logs prob looks like torch.Size([1864])\n", + "torch.from_numpy(rewards) looks like torch.Size([1864])\n", + "rewards looks like (1324,)\n", + "log_probs looks like (1324,)\n", + "logs prob looks like torch.Size([1324])\n", + "torch.from_numpy(rewards) looks like torch.Size([1324])\n", + "rewards looks like (1281,)\n", + "log_probs looks like (1281,)\n", + "logs prob looks like torch.Size([1281])\n", + "torch.from_numpy(rewards) looks like torch.Size([1281])\n", + "rewards looks like (1366,)\n", + "log_probs looks like (1366,)\n", + "logs prob looks like torch.Size([1366])\n", + "torch.from_numpy(rewards) looks like torch.Size([1366])\n", + "rewards looks like (957,)\n", + "log_probs looks like (957,)\n", + "logs prob looks like torch.Size([957])\n", + "torch.from_numpy(rewards) looks like torch.Size([957])\n", + "rewards looks like (1187,)\n", + "log_probs looks like (1187,)\n", + "logs prob looks like torch.Size([1187])\n", + "torch.from_numpy(rewards) looks like torch.Size([1187])\n", + "rewards looks like (1625,)\n", + "log_probs looks like (1625,)\n", + "logs prob looks like torch.Size([1625])\n", + "torch.from_numpy(rewards) looks like torch.Size([1625])\n", + "rewards looks like (1605,)\n", + "log_probs looks like (1605,)\n", + "logs prob looks like torch.Size([1605])\n", + "torch.from_numpy(rewards) looks like torch.Size([1605])\n", + "rewards looks like (1015,)\n", + "log_probs looks like (1015,)\n", + "logs prob looks like torch.Size([1015])\n", + "torch.from_numpy(rewards) looks like torch.Size([1015])\n", + "rewards looks like (1565,)\n", + "log_probs looks like (1565,)\n", + "logs prob looks like torch.Size([1565])\n", + "torch.from_numpy(rewards) looks like torch.Size([1565])\n", + "rewards looks like (1353,)\n", + "log_probs looks like (1353,)\n", + "logs prob looks like torch.Size([1353])\n", + "torch.from_numpy(rewards) looks like torch.Size([1353])\n", + "rewards looks like (1321,)\n", + "log_probs looks like (1321,)\n", + "logs prob looks like torch.Size([1321])\n", + "torch.from_numpy(rewards) looks like torch.Size([1321])\n", + "rewards looks like (1074,)\n", + "log_probs looks like (1074,)\n", + "logs prob looks like torch.Size([1074])\n", + "torch.from_numpy(rewards) looks like torch.Size([1074])\n", + "rewards looks like (1301,)\n", + "log_probs looks like (1301,)\n", + "logs prob looks like torch.Size([1301])\n", + "torch.from_numpy(rewards) looks like torch.Size([1301])\n", + "rewards looks like (2105,)\n", + "log_probs looks like (2105,)\n", + "logs prob looks like torch.Size([2105])\n", + "torch.from_numpy(rewards) looks like torch.Size([2105])\n", + "rewards looks like (2008,)\n", + "log_probs looks like (2008,)\n", + "logs prob looks like torch.Size([2008])\n", + "torch.from_numpy(rewards) looks like torch.Size([2008])\n", + "rewards looks like (1885,)\n", + "log_probs looks like (1885,)\n", + "logs prob looks like torch.Size([1885])\n", + "torch.from_numpy(rewards) looks like torch.Size([1885])\n", + "rewards looks like (1184,)\n", + "log_probs looks like (1184,)\n", + "logs prob looks like torch.Size([1184])\n", + "torch.from_numpy(rewards) looks like torch.Size([1184])\n", + "rewards looks like (2551,)\n", + "log_probs looks like (2551,)\n", + "logs prob looks like torch.Size([2551])\n", + "torch.from_numpy(rewards) looks like torch.Size([2551])\n", + "rewards looks like (1330,)\n", + "log_probs looks like (1330,)\n", + "logs prob looks like torch.Size([1330])\n", + "torch.from_numpy(rewards) looks like torch.Size([1330])\n", + "rewards looks like (1510,)\n", + "log_probs looks like (1510,)\n", + "logs prob looks like torch.Size([1510])\n", + "torch.from_numpy(rewards) looks like torch.Size([1510])\n", + "rewards looks like (1330,)\n", + "log_probs looks like (1330,)\n", + "logs prob looks like torch.Size([1330])\n", + "torch.from_numpy(rewards) looks like torch.Size([1330])\n", + "rewards looks like (2157,)\n", + "log_probs looks like (2157,)\n", + "logs prob looks like torch.Size([2157])\n", + "torch.from_numpy(rewards) looks like torch.Size([2157])\n", + "rewards looks like (1276,)\n", + "log_probs looks like (1276,)\n", + "logs prob looks like torch.Size([1276])\n", + "torch.from_numpy(rewards) looks like torch.Size([1276])\n", + "rewards looks like (1188,)\n", + "log_probs looks like (1188,)\n", + "logs prob looks like torch.Size([1188])\n", + "torch.from_numpy(rewards) looks like torch.Size([1188])\n", + "rewards looks like (2381,)\n", + "log_probs looks like (2381,)\n", + "logs prob looks like torch.Size([2381])\n", + "torch.from_numpy(rewards) looks like torch.Size([2381])\n", + "rewards looks like (1450,)\n", + "log_probs looks like (1450,)\n", + "logs prob looks like torch.Size([1450])\n", + "torch.from_numpy(rewards) looks like torch.Size([1450])\n", + "rewards looks like (1612,)\n", + "log_probs looks like (1612,)\n", + "logs prob looks like torch.Size([1612])\n", + "torch.from_numpy(rewards) looks like torch.Size([1612])\n", + "rewards looks like (1780,)\n", + "log_probs looks like (1780,)\n", + "logs prob looks like torch.Size([1780])\n", + "torch.from_numpy(rewards) looks like torch.Size([1780])\n", + "rewards looks like (1350,)\n", + "log_probs looks like (1350,)\n", + "logs prob looks like torch.Size([1350])\n", + "torch.from_numpy(rewards) looks like torch.Size([1350])\n", + "rewards looks like (1459,)\n", + "log_probs looks like (1459,)\n", + "logs prob looks like torch.Size([1459])\n", + "torch.from_numpy(rewards) looks like torch.Size([1459])\n", + "rewards looks like (1958,)\n", + "log_probs looks like (1958,)\n", + "logs prob looks like torch.Size([1958])\n", + "torch.from_numpy(rewards) looks like torch.Size([1958])\n", + "rewards looks like (1325,)\n", + "log_probs looks like (1325,)\n", + "logs prob looks like torch.Size([1325])\n", + "torch.from_numpy(rewards) looks like torch.Size([1325])\n", + "rewards looks like (2168,)\n", + "log_probs looks like (2168,)\n", + "logs prob looks like torch.Size([2168])\n", + "torch.from_numpy(rewards) looks like torch.Size([2168])\n", + "rewards looks like (1682,)\n", + "log_probs looks like (1682,)\n", + "logs prob looks like torch.Size([1682])\n", + "torch.from_numpy(rewards) looks like torch.Size([1682])\n", + "rewards looks like (852,)\n", + "log_probs looks like (852,)\n", + "logs prob looks like torch.Size([852])\n", + "torch.from_numpy(rewards) looks like torch.Size([852])\n", + "rewards looks like (1757,)\n", + "log_probs looks like (1757,)\n", + "logs prob looks like torch.Size([1757])\n", + "torch.from_numpy(rewards) looks like torch.Size([1757])\n", + "rewards looks like (2313,)\n", + "log_probs looks like (2313,)\n", + "logs prob looks like torch.Size([2313])\n", + "torch.from_numpy(rewards) looks like torch.Size([2313])\n", + "rewards looks like (1662,)\n", + "log_probs looks like (1662,)\n", + "logs prob looks like torch.Size([1662])\n", + "torch.from_numpy(rewards) looks like torch.Size([1662])\n", + "rewards looks like (1559,)\n", + "log_probs looks like (1559,)\n", + "logs prob looks like torch.Size([1559])\n", + "torch.from_numpy(rewards) looks like torch.Size([1559])\n", + "rewards looks like (2077,)\n", + "log_probs looks like (2077,)\n", + "logs prob looks like torch.Size([2077])\n", + "torch.from_numpy(rewards) looks like torch.Size([2077])\n", + "rewards looks like (2119,)\n", + "log_probs looks like (2119,)\n", + "logs prob looks like torch.Size([2119])\n", + "torch.from_numpy(rewards) looks like torch.Size([2119])\n", + "rewards looks like (954,)\n", + "log_probs looks like (954,)\n", + "logs prob looks like torch.Size([954])\n", + "torch.from_numpy(rewards) looks like torch.Size([954])\n", + "rewards looks like (1797,)\n", + "log_probs looks like (1797,)\n", + "logs prob looks like torch.Size([1797])\n", + "torch.from_numpy(rewards) looks like torch.Size([1797])\n", + "rewards looks like (1579,)\n", + "log_probs looks like (1579,)\n", + "logs prob looks like torch.Size([1579])\n", + "torch.from_numpy(rewards) looks like torch.Size([1579])\n", + "rewards looks like (1277,)\n", + "log_probs looks like (1277,)\n", + "logs prob looks like torch.Size([1277])\n", + "torch.from_numpy(rewards) looks like torch.Size([1277])\n", + "rewards looks like (1196,)\n", + "log_probs looks like (1196,)\n", + "logs prob looks like torch.Size([1196])\n", + "torch.from_numpy(rewards) looks like torch.Size([1196])\n", + "rewards looks like (1294,)\n", + "log_probs looks like (1294,)\n", + "logs prob looks like torch.Size([1294])\n", + "torch.from_numpy(rewards) looks like torch.Size([1294])\n", + "rewards looks like (1318,)\n", + "log_probs looks like (1318,)\n", + "logs prob looks like torch.Size([1318])\n", + "torch.from_numpy(rewards) looks like torch.Size([1318])\n", + "rewards looks like (2605,)\n", + "log_probs looks like (2605,)\n", + "logs prob looks like torch.Size([2605])\n", + "torch.from_numpy(rewards) looks like torch.Size([2605])\n", + "rewards looks like (2002,)\n", + "log_probs looks like (2002,)\n", + "logs prob looks like torch.Size([2002])\n", + "torch.from_numpy(rewards) looks like torch.Size([2002])\n", + "rewards looks like (1354,)\n", + "log_probs looks like (1354,)\n", + "logs prob looks like torch.Size([1354])\n", + "torch.from_numpy(rewards) looks like torch.Size([1354])\n", + "rewards looks like (1785,)\n", + "log_probs looks like (1785,)\n", + "logs prob looks like torch.Size([1785])\n", + "torch.from_numpy(rewards) looks like torch.Size([1785])\n", + "rewards looks like (781,)\n", + "log_probs looks like (781,)\n", + "logs prob looks like torch.Size([781])\n", + "torch.from_numpy(rewards) looks like torch.Size([781])\n", + "rewards looks like (1965,)\n", + "log_probs looks like (1965,)\n", + "logs prob looks like torch.Size([1965])\n", + "torch.from_numpy(rewards) looks like torch.Size([1965])\n", + "rewards looks like (1135,)\n", + "log_probs looks like (1135,)\n", + "logs prob looks like torch.Size([1135])\n", + "torch.from_numpy(rewards) looks like torch.Size([1135])\n", + "rewards looks like (1672,)\n", + "log_probs looks like (1672,)\n", + "logs prob looks like torch.Size([1672])\n", + "torch.from_numpy(rewards) looks like torch.Size([1672])\n", + "rewards looks like (1278,)\n", + "log_probs looks like (1278,)\n", + "logs prob looks like torch.Size([1278])\n", + "torch.from_numpy(rewards) looks like torch.Size([1278])\n", + "rewards looks like (2499,)\n", + "log_probs looks like (2499,)\n", + "logs prob looks like torch.Size([2499])\n", + "torch.from_numpy(rewards) looks like torch.Size([2499])\n", + "rewards looks like (1275,)\n", + "log_probs looks like (1275,)\n", + "logs prob looks like torch.Size([1275])\n", + "torch.from_numpy(rewards) looks like torch.Size([1275])\n", + "rewards looks like (1144,)\n", + "log_probs looks like (1144,)\n", + "logs prob looks like torch.Size([1144])\n", + "torch.from_numpy(rewards) looks like torch.Size([1144])\n", + "rewards looks like (1605,)\n", + "log_probs looks like (1605,)\n", + "logs prob looks like torch.Size([1605])\n", + "torch.from_numpy(rewards) looks like torch.Size([1605])\n", + "rewards looks like (1178,)\n", + "log_probs looks like (1178,)\n", + "logs prob looks like torch.Size([1178])\n", + "torch.from_numpy(rewards) looks like torch.Size([1178])\n", + "rewards looks like (3269,)\n", + "log_probs looks like (3269,)\n", + "logs prob looks like torch.Size([3269])\n", + "torch.from_numpy(rewards) looks like torch.Size([3269])\n", + "rewards looks like (1492,)\n", + "log_probs looks like (1492,)\n", + "logs prob looks like torch.Size([1492])\n", + "torch.from_numpy(rewards) looks like torch.Size([1492])\n", + "rewards looks like (1285,)\n", + "log_probs looks like (1285,)\n", + "logs prob looks like torch.Size([1285])\n", + "torch.from_numpy(rewards) looks like torch.Size([1285])\n", + "rewards looks like (1687,)\n", + "log_probs looks like (1687,)\n", + "logs prob looks like torch.Size([1687])\n", + "torch.from_numpy(rewards) looks like torch.Size([1687])\n", + "rewards looks like (1124,)\n", + "log_probs looks like (1124,)\n", + "logs prob looks like torch.Size([1124])\n", + "torch.from_numpy(rewards) looks like torch.Size([1124])\n", + "rewards looks like (2043,)\n", + "log_probs looks like (2043,)\n", + "logs prob looks like torch.Size([2043])\n", + "torch.from_numpy(rewards) looks like torch.Size([2043])\n", + "rewards looks like (1280,)\n", + "log_probs looks like (1280,)\n", + "logs prob looks like torch.Size([1280])\n", + "torch.from_numpy(rewards) looks like torch.Size([1280])\n", + "rewards looks like (1418,)\n", + "log_probs looks like (1418,)\n", + "logs prob looks like torch.Size([1418])\n", + "torch.from_numpy(rewards) looks like torch.Size([1418])\n", + "rewards looks like (1365,)\n", + "log_probs looks like (1365,)\n", + "logs prob looks like torch.Size([1365])\n", + "torch.from_numpy(rewards) looks like torch.Size([1365])\n", + "rewards looks like (1091,)\n", + "log_probs looks like (1091,)\n", + "logs prob looks like torch.Size([1091])\n", + "torch.from_numpy(rewards) looks like torch.Size([1091])\n", + "rewards looks like (1279,)\n", + "log_probs looks like (1279,)\n", + "logs prob looks like torch.Size([1279])\n", + "torch.from_numpy(rewards) looks like torch.Size([1279])\n", + "rewards looks like (1109,)\n", + "log_probs looks like (1109,)\n", + "logs prob looks like torch.Size([1109])\n", + "torch.from_numpy(rewards) looks like torch.Size([1109])\n", + "rewards looks like (1285,)\n", + "log_probs looks like (1285,)\n", + "logs prob looks like torch.Size([1285])\n", + "torch.from_numpy(rewards) looks like torch.Size([1285])\n", + "rewards looks like (1222,)\n", + "log_probs looks like (1222,)\n", + "logs prob looks like torch.Size([1222])\n", + "torch.from_numpy(rewards) looks like torch.Size([1222])\n", + "rewards looks like (1538,)\n", + "log_probs looks like (1538,)\n", + "logs prob looks like torch.Size([1538])\n", + "torch.from_numpy(rewards) looks like torch.Size([1538])\n", + "rewards looks like (1139,)\n", + "log_probs looks like (1139,)\n", + "logs prob looks like torch.Size([1139])\n", + "torch.from_numpy(rewards) looks like torch.Size([1139])\n", + "rewards looks like (1354,)\n", + "log_probs looks like (1354,)\n", + "logs prob looks like torch.Size([1354])\n", + "torch.from_numpy(rewards) looks like torch.Size([1354])\n", + "rewards looks like (1166,)\n", + "log_probs looks like (1166,)\n", + "logs prob looks like torch.Size([1166])\n", + "torch.from_numpy(rewards) looks like torch.Size([1166])\n", + "rewards looks like (1348,)\n", + "log_probs looks like (1348,)\n", + "logs prob looks like torch.Size([1348])\n", + "torch.from_numpy(rewards) looks like torch.Size([1348])\n", + "rewards looks like (1347,)\n", + "log_probs looks like (1347,)\n", + "logs prob looks like torch.Size([1347])\n", + "torch.from_numpy(rewards) looks like torch.Size([1347])\n", + "rewards looks like (2059,)\n", + "log_probs looks like (2059,)\n", + "logs prob looks like torch.Size([2059])\n", + "torch.from_numpy(rewards) looks like torch.Size([2059])\n", + "rewards looks like (2021,)\n", + "log_probs looks like (2021,)\n", + "logs prob looks like torch.Size([2021])\n", + "torch.from_numpy(rewards) looks like torch.Size([2021])\n", + "rewards looks like (2232,)\n", + "log_probs looks like (2232,)\n", + "logs prob looks like torch.Size([2232])\n", + "torch.from_numpy(rewards) looks like torch.Size([2232])\n", + "rewards looks like (1102,)\n", + "log_probs looks like (1102,)\n", + "logs prob looks like torch.Size([1102])\n", + "torch.from_numpy(rewards) looks like torch.Size([1102])\n", + "rewards looks like (1165,)\n", + "log_probs looks like (1165,)\n", + "logs prob looks like torch.Size([1165])\n", + "torch.from_numpy(rewards) looks like torch.Size([1165])\n", + "rewards looks like (1264,)\n", + "log_probs looks like (1264,)\n", + "logs prob looks like torch.Size([1264])\n", + "torch.from_numpy(rewards) looks like torch.Size([1264])\n", + "rewards looks like (1346,)\n", + "log_probs looks like (1346,)\n", + "logs prob looks like torch.Size([1346])\n", + "torch.from_numpy(rewards) looks like torch.Size([1346])\n", + "rewards looks like (2848,)\n", + "log_probs looks like (2848,)\n", + "logs prob looks like torch.Size([2848])\n", + "torch.from_numpy(rewards) looks like torch.Size([2848])\n", + "rewards looks like (938,)\n", + "log_probs looks like (938,)\n", + "logs prob looks like torch.Size([938])\n", + "torch.from_numpy(rewards) looks like torch.Size([938])\n", + "rewards looks like (1069,)\n", + "log_probs looks like (1069,)\n", + "logs prob looks like torch.Size([1069])\n", + "torch.from_numpy(rewards) looks like torch.Size([1069])\n", + "rewards looks like (2588,)\n", + "log_probs looks like (2588,)\n", + "logs prob looks like torch.Size([2588])\n", + "torch.from_numpy(rewards) looks like torch.Size([2588])\n", + "rewards looks like (1461,)\n", + "log_probs looks like (1461,)\n", + "logs prob looks like torch.Size([1461])\n", + "torch.from_numpy(rewards) looks like torch.Size([1461])\n", + "rewards looks like (2153,)\n", + "log_probs looks like (2153,)\n", + "logs prob looks like torch.Size([2153])\n", + "torch.from_numpy(rewards) looks like torch.Size([2153])\n", + "rewards looks like (2312,)\n", + "log_probs looks like (2312,)\n", + "logs prob looks like torch.Size([2312])\n", + "torch.from_numpy(rewards) looks like torch.Size([2312])\n", + "rewards looks like (1636,)\n", + "log_probs looks like (1636,)\n", + "logs prob looks like torch.Size([1636])\n", + "torch.from_numpy(rewards) looks like torch.Size([1636])\n", + "rewards looks like (2019,)\n", + "log_probs looks like (2019,)\n", + "logs prob looks like torch.Size([2019])\n", + "torch.from_numpy(rewards) looks like torch.Size([2019])\n", + "rewards looks like (1450,)\n", + "log_probs looks like (1450,)\n", + "logs prob looks like torch.Size([1450])\n", + "torch.from_numpy(rewards) looks like torch.Size([1450])\n", + "rewards looks like (2105,)\n", + "log_probs looks like (2105,)\n", + "logs prob looks like torch.Size([2105])\n", + "torch.from_numpy(rewards) looks like torch.Size([2105])\n", + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vNb_tuFYhKVK" + }, + "source": [ + "### Training Result\n", + "During the training process, we recorded `avg_total_reward`, which represents the average total reward of episodes before updating the policy network.\n", + "\n", + "Theoretically, if the agent becomes better, the `avg_total_reward` will increase.\n", + "The visualization of the training process is shown below: \n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 281 + }, + "id": "wZYOI8H10SHN", + "outputId": "54840043-6fe4-4771-e8c9-78785c55aa79" + }, + "source": [ + "plt.plot(avg_total_rewards)\n", + "plt.title(\"Total Rewards\")\n", + "plt.show()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEICAYAAAC3Y/QeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOx9d9wdRb3+M7unvD1vOglJSCChhA6hKF2Rjnj1ykW8XGwXC3ivehXhooIgyrUXwJ+oKCJSFEEUkCZNlJJQQoAQQnqvb9562u78/tid2dnZmd0973vOW+f5fJL3nN09s7N7zj7znedbhlBKYWBgYGAwtmANdQcMDAwMDAYfhvwNDAwMxiAM+RsYGBiMQRjyNzAwMBiDMORvYGBgMAZhyN/AwMBgDMKQv8GYByGEEkLmDnU/+gtCyImEkHVD3Q+DkQVD/gbDFoSQbuGfSwjpE95/WPOZmhIhIeQJQkjBP+c2QsgfCSHTatW+gcFQwZC/wbAFpbSF/QOwBsDZwrbbBrErl/h9mAugBcB3B/HcIRBCMkN1boPRBUP+BiMOhJA8IeSHhJAN/r8f+tuaATwIYLowQ5hOCDmSEPJPQkgHIWQjIeR6Qkiu2vNSSjsA3AvgEKEv+xJCHiGE7CCEvEkIOdffPsc/n+W//zkhZIvwuVsJIZ/zX3+UEPIGIaSLELKCEPJJ4bgTCSHrCCFfJoRsAvArQkgjIeTXhJCdhJDXARwh3Z8vE0LW++29SQh5d7XXajD6YcjfYCTiCgBHwyPhgwEcCeArlNIeAKcD2CDMEDYAcAB8HsAkAO8A8G4An6n2pISQiQDeD2C5/74ZwCMAfgdgCoDzANxICJlPKV0JoBPAof7HjwfQTQjZz39/AoAn/ddbAJwFoA3ARwH8gBBymHDq3QBMALAHgIsAXAlgL//fqQAuFPq4D4BLABxBKW3196+q9loNRj8M+RuMRHwYwNWU0i2U0q0Avg7gAt3BlNJFlNJnKaUVSukqAD+DR75p8WNCyC4A2+ANIJ/1t58FYBWl9Fd+2y8BuBvAB/39TwI4gRCym//+D/77OfCI/hW/f/dTSt+mHp4E8DCA44TzuwCupJQWKaV9AM4FcC2ldAeldC2AHwvHOgDyAOYTQrKU0lWU0reruFaDMQJD/gYjEdMBrBber/a3KUEI2ZsQ8hdCyCZCSCeAb8Ij8bT4L0rpOAAHARgPYIa/fQ8AR/nyTgchpAPewMTI/kkAJ8Kz+p8C8AS8QecEAE9TSl2/f6cTQp71paMOAGdI/dtKKS1I179Wun4AAKV0OYDPAbgKwBZCyB2EEO29MRi7MORvMBKxAR7xMszytwGAqkztTwEsBTCPUtoG4H8BkGpPSil9FcA3ANxACCHwCPhJSmm78K+FUvpp/yNPwrPgT/Rf/x3AMRAkH0JIHt5s4bsAplJK2wE8IPVPvqaNAGYK72dJ/fwdpfRYePeIAvi/aq/VYPTDkL/BSMTtAL5CCJlMCJkE4GsAfuvv2wxgIiFknHB8Kzz9vZsQsi+AT6P/uAXAVADvBfAXAHsTQi4ghGT9f0cwXZ9S+haAPgD/Dm+Q6PT79wEEen8OnkyzFUCFEHI6gFMS+nAXgMsJIeMJITMQyFAghOxDCHmXP6gU/PO7A7heg1EKQ/4GIxHfALAQwGIArwJ40d8GSulSeIPDCl+KmQ7giwDOB9AF4OcA7uzviSmlJQA/AvBVSmkXPKI+D97MYxM8KzsvfORJANt9bZ69J36f4bfxX/AIfaffz/sSuvF1eFLPSnj+gVuFfXkA18HzT2yC54i+vB+XajDKQcxiLgYGBgZjD8byNzAwMBiDMORvYGBgMAZhyN/AwMBgDMKQv4GBgcEYxIgoEjVp0iQ6e/bsoe6GgYGBwYjCokWLtlFKJ6v2jQjynz17NhYuXDjU3TAwMDAYUSCErNbtM7KPgYGBwRiEIX8DAwODMQhD/gYGBgZjEIb8DQwMDMYgDPkbGBgYjEEY8jcwMDAYgzDkb2BgYDAGYcjfwMDAoJ+496X16ClWhrob/YIhfwMDA4N+4KU1O/G5O1/GV+9dMtRd6RcM+RsYDGM4LsVX7n0Va7b31qX9HT0lPPbG5rq0PdrRU3QAAJu7CglHDk8Y8jcwGMZ4ZV0HfvvsGvz3nS/Vpf2P3/ICPn7LQnQWynVpfySgo7eEtTuqH1ypcrnokQND/gYGIwBunXhmxdYer/16nWAE4KTvPoHjvv141Z9jiyASkBr3aHBgyN/AYBiD00qdlls1y7gCO3v7N+thd46MTO435G9gMJxBfGapF0Ub6h+7MORvYDCMwYzKuhnofrvOGJZ9xioM+RsYDGMwSaFezkXWquH+6jHSJTND/gYGwxjMmVgvnmEE5o5wIhtKkBEq+hvyNzAYxuCWf5252cg+1WOk3zFD/gYGwxiB7FMfBLLPSKeyIQAP9RyZMORvYDCMEcg+9Qr19P66bl2aH3X408vrsXxLF4DADzNCVZ+RsYC7gcFYRb2JhRGYsfzT4b/veBkAsOq6M/m2Ecr9xvI3MBgJqJ/D1/vrGPKvGrW+ZY+9sRm7+plw1h8Y8jcwGMZgFnndQz2Nw7dq8PIONZiebekq4OO3LMTFv3txwG2lRU3InxByMyFkCyFkibBtAiHkEULIW/7f8f52Qgj5MSFkOSFkMSHksFr0wcBgNIIRTL0Nc8P9yZD9Lry8Qw3a7it5FUJX7+ipQWvpUCvL/9cATpO2XQbgMUrpPACP+e8B4HQA8/x/FwH4aY36YGAw6sDJv24n8P6YUM/k2Y98j9hgUAu/DGvbHkTvcU3In1L6FIAd0uZzANziv74FwPuE7b+hHp4F0E4ImVaLfhgYjDYwuadu0T7G4cuRdA8qEvnXcrxkbVnWCCN/DaZSSjf6rzcBmOq/3h3AWuG4df62EAghFxFCFhJCFm7durWO3TQwGL5w62z581BPQ/6JTu8o+dfunrG2RpzlnwTqmS1V3SlK6U2U0gWU0gWTJ0+uU88MDIY3mMVft2gf/+9oVH3uX7wRF/1mYerjg5wHqpSAHCe8LZCBBk7YFb9texAt/3rG+W8mhEyjlG70ZZ0t/vb1AGYKx83wtxkYGEjgln+dLfPRqPlXGznD7sG8rzyIeVNa8NfPHR/aX5Ey4dwaav5BW6PD8r8PwIX+6wsB/EnY/h9+1M/RAHYJ8pCBgUEIVPi/Dq2bwm4c7B44LsXSTV2R/YMh+2Qky3/jrj5s6Oir2XlE1MTyJ4TcDuBEAJMIIesAXAngOgB3EUI+DmA1gHP9wx8AcAaA5QB6AXy0Fn0wMBiNCCz/+rRv4vwDJJW4kMnf8Y+vha3OZh2yw/fLd7+Kzr4y7r34mBqcJYyakD+l9EOaXe9WHEsBXFyL8xoYjHYEoZ71re1jMnyTLXlZ82cDZi1lH1tqi1KKerkBTIavgcEwRhwhrdrWg3lXPIC3t3YP+DyjmfvT+kuSo300mn8dHb6OS2HVyQ9gyN/AYBjDjYn2+fMrG1B2KP744roBn2c0OnwZ0g5s1cb513K2VOGziDDRu5TWLfbfkL+BwXBGjObPSKEWHDRSZZ/7F2/E7MvuR2dBXxAt7bW5bvwsoVJH2aeiyfB1KYzsY2AwFhFnkDOeqIXRPpLWo3Vcysn+xieWAwBWb+vVHp82KselFMWK3usrz46cWpK/7z2WZR/XyD4GBmMTceUX+rPQS8Vxcf/ijZHoHmcELeZy7f1v4KCrHkah7HBijHOIp709jhtP/uWI5p+u3TQoO+poH5fSuiV+GfI3MKgDlm3uwpf/sHjAWnpcqCfjhGrO8McX1+Pi372I3z63WjrPyLH8b39+DQCg5Lj8HsTd5rTfAaVAseKkbqeW9ywo7Bbe7tL6JX4Z8jcw6AcKZQelGCvxP3+zEHcuXIu1O/RyRBrw8g4KimdWbzUx+syKfH5luA7jSIrz7yt7BE3dgBjjiDgtSTuUoljWf6ey5s9ln1pE+7hq2ceEehoYDDPs+9W/4j0/eFK7v6foEVRjzh7QeeLq+fdH8x/fnAUALN8SDg8diQ5fh9JggftY8k/XXrWafw1L+/CBRdb3HVo/zd+s4Wtg0E+s3q636ntLFQAD5wUaU94hjdUrg2n7MvmPIMOfQ4yBj+t/Wp+IW7XmX8tQT53DNzog1AqG/A3GFJ5dsR2HzGxHQ3ZgFnkSev2VmQZCD5+9/SX8+ZUNXjsqy9//W43DlxFWxaX408tBPcWRJPswuJTy0Mi4/qfV/F2KeMtfK/sMHCzUU+XwNbKPgcEAsXxLF8676Vl8/c+vDdo5B2IdMuL3EG2HtV0Nb4sDxW3Prom0JeOHjy7D7xeuVe4balRcypk37h6kvT9etI/e4Rut7ZNciXNXbxkdvaXEc+tkH7eOso8hf4Mxg45eLzb8TUXFxnqhVga1yM2/eHoFjrz2UR4eWM0Aw/ozpTUfkjF01vF9r2zAY29sUe4bangx8N7ruNlPatknQfOXyzsEay3o2z/46odxyNWPxJ73H29vw/WPe/kKqmifeoV6GtnHwKBKVCOz1Cp5SmzlG/e/ASBIDKpmgGEDRda2QoSvG0Bclw5bZ7Co+ctWeei4asg/JtpHvF+UBvdloLfn/J8/x1+rZJ96lfg3lr/BmAGPDElx7IaOPvzy7yuV+8pONeSf+tCEdqINlRn5V8H+jMDyWSt0HbomKi4dVnV/xPsgRsLE9TF9tE98nL8Y6ll2KHee19LxGynvYDJ8DQxqgfQP0cd+/QKu+cvr2NJZiOwrxBCEjFoRA2vlZmFAKvusVqoiPZd1J2dbfOYA6MnTFcj/intexbcefKOKXtcezJEOeH1jvFiOuQdpB0fHTW/5Oy4VfC41JP+I5V8/2ceQv4GBAuv91ZNUckKhlJ78a2f5A5t2FXD1X17n28q+Ph2XbCaDEVU+Y4WuTSdPiZb/bc+twc+eXFF132uJrkKFv05v+afN8I13+Io+krLr8kGllhMj2cg3so+BQQ2Rhgv6fIIvlKNkUIixDmXUzPKnNNIWm4Fs6y7itB8+laq0MyOqXMYKWctay58OL9mnS6je6QgO33INZB+xto/K2hbvQ8UJNP9ahsnKq4lRWr84f0P+BmMGSc/Qls4CHn5tE4DA4lcRfZLsE3akVtlJDSiiJMAkiudW7sDSTV144NVNie2wASSXCTt8dW6MajX/bd1FrNzWk/r4atFVFCx/l/IwSydmDcb0VT0RS/6i5l8JWf61I3+5jIc4wNUahvwNDHxc+KsXcNGti0LWvoroVbMBET2livCuduwvE4Mclji1LZ/YDCOsnB12+OpkH0cR7fPF378SssBF/OCRZfjELS8AAG54fDne9d0nEvtUDUTJLWT5xzjhqwr19L9b2fEKhEM9Q5Z/DSdGclumqqeBQQ2he1Y3+87dbsG6VBF9X4Lm3y3o0rW0/OXCYrLWH+f0lPuTy1ghfVtn3Tsujfg9/rBoHX7zz9XK47sKFez08ym+89CbWFHjWYA4GHt6uB/qGUP+8m35x/JteN8Nz0TulxjnrwoPrUiyD3v74pqduKtGiXDyLMJU9TQwqAGSDMDmvFfyQSRvpeaf4GDtKYrkH53GP/3W1qSuRqDS3uVZSRrHbyD72KGZg26Qclyq1LR1fOS4NDI4phmUVNjeXcSS9btC28RoHMcNylpXI/u8sm4XXl7bEXIee20E5K+6Zkcj+3QVKrj0D4tTXFEKKCx/I/sYGKTA029txZPL1OSapM0257ycxx1COr5S80+QfcTQS/mU1/9tOS745fP4+1vbYtuQQWnUGpXDEtPkHzAJJGdbocFCd29Ulj+glkUAjxT7yk6IPLslkk0D16V4343P4Kyf/D20vVCRZR+vH3HXLl8bKwktD0qUBjOgRMtf4QtJ+l2kQdTyp9p7PVAY8jcYVbjgl8/jwpufV+7jhKQhupa8R/5iDX51tE/8Qy42Lz/MyzZ7pSV2pKj3EmoTUbKRwxLjShME/fH+5jLhR18r+1C15a/ToVk7Yl9EGS0NdvWWsef/PoC1O7xwW1GzFwdjse5NbKindFvY9yfPlMTYfUqjvgJR8y87bmSA2NZdjL2uNHCpl2C4aVfB7zsdfbIPIeQ0QsibhJDlhJDLhqofBmMHSWn+zT75r9kukn+85a9yJoqEL+9ms4KcXf2jJ+vaMtmnkVd4hq9E/irL33UpKI3WtAH0OjSzjnsFp3fc4uoqbOkKJ9aJ11kshy1/nuRVhezDZKmS44a+P1cKp5UHFNHyd/x7I2JrVzz5P/r65kTfAAXwzuv+hqO/9Zj3frSFehJCbAA3ADgdwHwAHyKEzB+KvhiMHcTwA4DA8hctuKQ4f5XFGa4BE97HrE2ZfJMgShIM/SF/MdQztF11HTHRLHIBMv4Z/+C+soOGrHeOamUfmevErF7R3xKq7dNP2UeubyT+RmRjwYmUd5Atf/1sjlKK3z2/Br98eqX2GFVfnVGo+R8JYDmldAWltATgDgDnDFFfDMYI+JRes585fDtFh29CqKeshz+3YjseXxpUwfz9orV4ac1O/p6Rv0y+SaBQaP5C33IZK5XDVyzvIEJF8IzcKq4bITq5ABkDI+FC2eFrJlQr+8izCnEWUdBY/vL3ELbow+1z8q/Q0D6XhslXNhbC0T5R2SfO8q+4FGXHTRyg5ZlkPUM9h6qq5+4AxPnPOgBHiQcQQi4CcBEAzJo1a/B6ZjBqkST7NPkO386+QKYQ48p7ihVkbMLJA/CsR3FhmH+76dlQm7/552oUyy4OnTUeQCD7VP0402hEi+jwndKaH5Dlr7o3jPBdNzqr0PXf4bKPw2c3clRNtRCjh0KzLiECquJE9XuGSGY0l32ciMwj3gf5nlSkEtgyUceSv0NRqriJfhn5axiToZ6U0psopQsopQsmT5481N0xGAVISsPP+lpGV8jyDx7W/a98CO/9yTMhAoqTG/h5hSe6FBNHHgcKGqv5T2nND8jhq9L8K4LlL5O/zsHKCLKvFFj+XVVa/nLbvSUHb/mOcnG24woO2jh9Xv7emYxUqtCIc158r1u5C/DKSURlHz35l10XpRSWfyTaZxRm+K4HMFN4P8PfZmBQNyQE+/D9u0TLX9L839zcFYrjj3M0yu0CAfnH1Z9XQa35B32b1FKl5S/LPor+sG2OGw2l1IVWcsu/7KAhE82bSAN5kLvjhTV4zw+ewt+Wbo5Y/uz2y/2JK7Ehav4hmUd2+MqWvxjn77iR5DEWwXX3onX465JNoXvqOJ7sI1dg1S4KL/RptGX4vgBgHiFkDiEkB+A8APcNUV8MxgiSatSwB1+MTlE5fJdu6uSvP3/ny3ww0JGvKA+wY2SrMgkujZZZKFZcnH7Abnjj6tPQnM+kKu3MyztEQj2jx1Y4+bsRWUU36PGaSCWH+wV0pSB0kL+nhas8n8mbm7oj0T4ODfqo6gegkH1Eh29I9gmTb9xsouxEC+2xKqv/8/tX8KnfLgof77ooVyg/JmgnmmsgYtTJPpTSCoBLADwE4A0Ad1FKB29hVYMxicDhqyZe9uCFNH9FqOeS9Z1oa/D8A88s346b/74SfSUH2zXRHiJJxJUPiAOFyqnpkXhjzkbWJihX0khQ3l+V7PPGxk585rZFwSIxgqQiDyy6c4maPxswqnX4ygMLc7pTeBm4TJ5zhRwEuapnnOYvWv5UOJXYHgD8+LG3sKuvzB3O8oAit6tb4xfwNX+F5S9LdSFHtf/50Sb7gFL6AKV0b0rpXpTSa4eqHwajE0oZQ0jgUYE9eD0hB6Oitk/ZweF7jOfvv/fIMiz4xiNYuHqHui+i7MMs/4RZiOxMpFQ9W8hY3iOcy1jpLH+/XVWc/xfuegUPvLqJr3EcWP5Rf4Mq9l/8TF/Z4a/Tyj6vrO3w5RSJrP3vg1Lv+2CO+YpgfVf8mP2dPaVI/yKyj99esSLJPm6Y0G99djUO/nqwBq/jujy6yFFo/rIVLzuISxUXZYcqZ4IMYousLybD18CgCqiIMJFwFdv6NNm886e3hd73lBxc8ruXlMe6ioc9ri/v+NZj+PZDb0a2q2YLzArO2lZEUlCBUi88kg0aYh9zNiuVEK5v41AaISndQMPj/EsO92+kcfguWb8L59zwDH7w6LLIQBOK9qk4aMp5voQ/vrgeb2wMBqq/L9+GI7/5KLZ2FVNa/mGyl0M9+bUyP41Dua/EcRWyj0z+Tvh7Z/vFeyeH58r9AfRhtQOFIX+DUQmV/p5U20feb1sExbKLjt4SfvtsuIolsz7TQGyWR/vEkP/GXQX89Im3I9tVn8nageVflK75zU1d+NvSzeE2/JIIshPRcYO2ShUXL6zageO+/Tg/r+xQ1UU58WifssNfs35TSjH7svvxnYeWRj63bqeXVb1sc3fU8i87/PPFssvJ/58rtmOTX4m14lBs7iyi7FDs6iuF+ifPArWaP6VK30dwbZTLZSrLPzo7oqHXjPTLTtQYCKSs4PPs92hW8jIwqAKqhCc2CxfJuFhxsMOXCmRubczaKFQcfPnuxfjKvUtC+6rJ0FWGelYZ7QOoI4QyPml49fnD5QpO/eFT+NivF0p98TTkrJSi6wrEVqy4+NGjb/F9jhu1/HXObSZN9ZYcTnJBspj394bHowMbI/imnB3R/MXLLlQcXoYjdF6XcqevXHRN/LxngVP+WvwtUBqN3RdRcSnyfgSTakCMC4ctO65y2U32+vvnHoJDZ7VHyk0Ao6y8g4FBvaGUfRQP9md++yIOu8bTdOXdrQ0Z7Oor88FBRDUZuiL5izp6tVCVLeaWv20pw0FVfbEIQSaS4Ut5W72lSkR+SEv+wQpoTkTiisuJYLH3jVlbGwnlUs8B36yYdZUdN8hLcKg22kf04ZQkzV8l5YTO7wbS2KV3L8ai1TtD+yOzI0nzFwcdhqKQ8W0RIuUdeH+N5m9gIMFxKRZpnKyqaBSVE/gxvxSDq8jYnDulBWt39CqTp3SF2eZNaYmeV8En1Ub7AOpwTK75+4NRktOXFQrLyrIPDSz/nqKjKFecrnx0EO1T4STHSDAuJ4Lp+o05W5sDUaq4vsPXjuwTrX1ZkhG/d9GHU3LcSHmHuAhcR5gdqSA7wWXLn303ouXP7lHOtkAga/5G9jEwUOJnT72ND/z0n/jn29sj+0pO1FHLk7wUbfWVnYjlv9+0NrgUWLE1uhqVjgSyikFBJSVUm+QFqC1/Hu3jnzcp3JNljMqWP6VBGz2lSuReyFFPSZZ/X9nlln4ay79PsPx1kUSFsoNixUWTQvapCBFJFdfVRvsUSgLxVsKhnSoDgIHlFKi+X95ejObfJ5xX5fBllr9q0DKyj4GBhLc2dwPw6p/LUFnrcdZ2b8mJTPn33a0VgDpOXU/+0QeVPc/iCl66UhNJmrPufMzyLyoGPbkvnuwT9DNre6TD2uouViL3Qi5wpyNybvkXKxGJS04UEyEm1umkq0LFQaHsoFll+QshomUnHJoqXktvWcjOTsjwFdFbqsB1k8g/fH13L1rHX/dJclPwGe98uYwFQsJtcNlnlBV2MzAYMJhBFBeeJ0JFuPmMhWLFRV/Jicgzsyc1a6tl6mQf2aIGAkK/4JfBIjM6yz9uQqAiXJtb/skrWnnts1DPgFDaGrJwKOV971GRvyT76OQlZnGL9ZE4KWsu7oePLsPP/VLHpYqrHVgKZRfFsotGjezD7ul5UnE98VrEsFFZ9hELxcnoLTme5R8n+0gx/DcKEVtiZVKR4NkMNWsz8g8+z3xRoy7Jy8BgoGDTYfa8hZNnFJo/T/IK9jELvrdciWT+Zi0LbQ1Z5bl1ln9G8aQ6NFr0S2f5xzlsVVILs9ZZf1QDlRxBYlskFOff2pABpYFDsqcYHQhl2UdnxbP+i5Z8JcHy/6EQWVRy3FjZp+yGq6iK/dGt4xsi/xiHL6X6wbe35C1NmU+w/HWDurgmgSraJ2d7so/qOx5V5R0MDGoB9kiwBziUSauSfZSWv0ckvaWo5k8IoHvWRfIXX6tkgaeWbcWCbzwKADjzoGkA9JZ/HPmrrG0mCbDzqshDjnyxCAnJU/mMjR09JV6jSG35y5q/up/sXGKJDFGOSUIcgRZ8v4zqHqtCLxmWbgyK8RWkctziwKgq08zQW6r4lr+eiMuOftbSK804gtdM9iGwCFF+x0bzNzCQwB4KlwLrO/pw2g+f4vvUDl+17AN4coD84BOiD7MT4/zPOnAaf63S/FWfcynFss1dmH3Z/VixtZvvj/NLqAY0NtPI2WHLXyY1Bpd6lqQoT1kWwbMrduBvfuRTTylK/mlWDWPLPgLhBXHERWFU2GNiU+g8ugGwp+hLJIrZlWplLYafPbUCn/rtIgBhx2s5Eu3jLe6i0th7S04oEU4FMZFLRp8g+6gtfzui+TP0Y8XPVDDkbzBiwZQLCopfP7MSb20JSLSkCvVUcAMj416F5m8Rok2tz9mB9CAeo9L8VeerOBT3vORVMX9wySa+v1rZh51PDvUsKsIJAW9QsCTNXx70uouOdtHzPSc1a/siWuyikzwp2mfm+ID8Wf0bFXp8ArUVA6wX4aO/dywijMk+FokOGK4fIqoawJnsE7f2smf5q8lftPyVGb4ZAkKI8h4Z2cfAQAIRLH/5mUtb24dr/gprV3aMqj6XsUjIIZdk+TPL0aGUy1Y6K12GihSZlZrnoZ7edYtkIxIKW/dWjPbpKYWjmeJkn99/6h04aZ/Jyr7oF3hhsk+8Jp/LWP66usFx7U2Bz4VJN6rZWEXI8I0DI//WhmxE83ep911krSgtetFLbmKoZ1rNn12LqPkTqO+RkX0MDCTY3OEbDdFLHe3jOw/7Sk4k/j/W8vfJP2tbIZkgjhwALy7ftkioQqTY9VjNX3FNuiQvMbqkIsk+ssNXXmaxp6iK8/fazdiWV0ROafkHMeti/+TyDtHPURw1ZwL2mdrqRfsIx01ozgn98ghUJcvIWb06sCUcxzVmUZLKOzi+7JPRWf7Um+Wpzs++y//3ZLR0BRCOMrrxieXY/8qHsLWrGCR5ZSxYRP0dG9nHwEACewZVpK4kfx4VFGxjFnOvQvO3YjT/DHe0kpBlJlfLjHzOJj75q/fHlRdQ68FSkpfjoqtQxrH/9zg/Jpzw5IV6ijMU0TkLeDX/RbYAACAASURBVNaxzvLPxZA/I3m21gEANGRsweEb/syGjj7sfcWDWLyuA1nbQi5jYX1HH15dt4sfM6FJIH8m+6giqly95i9fG+BFOD3y+mY89FogubmUyT4Ky79UgeNS2JrfRKNvRPzqmVXK8/YKjubXNniLAW3uLHB5LsujfaLXUC/L38T5G4xYMNnnz4s3RhbPVpITC/UUbPx81nf4KjJ8AbWVB4DHmp+7YGZIYsrFRIMAHnHZxLP8M5bt90foY5WWPxuEWPhjb8mJZCSLsg8r7yD6JuRTVhwK25LIv8Isfy9SSGVls20t+Qy2+QvbNAjlGmT56YFXN3r3zvHuS9YmWLa5G8s2B76b8SHLvxK6ZhGOG2/5sz19ZSek2//kb8v5a1bPX0X+X/3Ta7AIsGD2eO83IcUTNGTtkLQjo6+kLmsdkn00Dl+j+RsYSGAW0aLVO7FmR29oX5zsIxIse7BUmr9F9FZXcy6DV686Bf97xn7VWf4WQca3/KuVfZQOX58IW3xru6dYiQxY0VBPvS+DnUdXBjljeQOHau0A1vdWITeiIWsJ1TaDzxTKDrYLBfMyFkEuE43fHy9o/qxLKinOK++g1/zZrM5bWN7C6u29kWM8zT8s+3z7Xw8K7beFctjv3Gsi/vXwGd51JhT6Uw0MXuVT1/MbWb7DV/H9m8JuBqMWX7jrZXxXsXhJEuIyH29+ZmUkNl2VD8CISRXtQxR174N9HsnJGrBKLxZh+w+647qBwxdhi1gHlRObna/Fr3fTVahEjhNJkTt8Y25exY2uF1woO8jaHkFlbUuZrSta/gwNGVtw+AafufXZ1dguJL7ZFuFZyiJaFUl2KjJ0EqJ9xOtozNnKkh2q+j3nLpiJH513SKif7NZNbMlz/0aDIutYhIr8e0oVlCoub0P3lZgMX4NRiz++uB7XP748+UAJcc/6xl0FPLVsa2hbsNh3lGz7FEleFtHXVRGtffGYuFBAwNN2M5Zv4fltVBwa1PmPjfOP7mMzDUa43cVKZNYjkiKVnJb7TG2NtFl23EgbxXIQ6ZK11ZmorBRzi6j5Z20+ixBln+seXIp1O4OaTBmbKLOmWxuiyrTS4Zug+bM9vSWH6/NA2KHMAgdk2WevyUGlVvHeZSzCZxQNilmLiD4V+Re9yqfsugl0hoax/A3GEO5auBbzv/bX2Kl8UmhfpxTFopJ92Ov7F2+MlGAgMZq/uDkk+6Sw/G2LhCSm6x9fjsOvYevExss+k1pyePbydwfn8ztiWwRNORvdBQX5S0XOLOIRyt2ffgfuuOjoyHkqDo0kdfWUKjxHQbVkJKVB3oJI2F6VTnWS19qdgfRiW5Zy4FTV7tc5fGM1f39XX9lBQ9bG+UfNAhCW0hxK/QzicPvid+rJPhbvB2u3IZsg+5SjM43eooOSEwyqOsXQWP4GYwpfv+819JYc5fScQVco7BvvOwAAsEuKYmGHq8i/q1jBy2s7QsfHZfiGLf9gexrN37aiyTxsndsk8vdko2CbSIQt+UzI8v+vd8/zPueT7ru+9wQeXLKJ9/3wPSaEHKoMjksjkpmnlXvWbcYmkXv//Mod+MGjywAArYLsk89a2vIOIcvfIkpH67jGLN+vumaGiku1i8CIYLLPN//lQBw8Y1xokNvZU1bG8svnZrttEgziqmJzInqLatmnWHH5oKez8OtV1dOQv8GwBCMoFfnf+9J6fP7Ol7UP+/imHAgBdvWGV+Dii32kTKqyJKIVQTSWf9IKX8zyd6jaQZkU7WMTEhqQxKidloYMuoqB5s9CLlmbLApIJpmJ0gBQdt1Yyz+nCPUUNW1Rp89nxGgfebYQvLYtteyTy1hYfu3p+NKp+4SOlZFk+QPA429uQZ8g+2TscMXW+1/diLU7+jjZs9tkCz8CS7j/tk24QZEk+6gWsWdLXQayjxomyctgTIFFdMgJSABw6R8W456X1kcsdYaMTdCazygsf9/hK1r+lOJd+05RtkOgJhpCwgQajvaJf1CZ5e+4NHXxOYbuYgV2JK9AsvwF2YctMi8TtdzFp798Ei45aS5/T2l0ScveksOL4GX9JSPFdkMzEEH2YZa761LtTI1dhy5BLmNboUFOH+oZf50f/dUL6CpUOPnrsrFZP9h5wpZ/8NvMCPJd0qDPvpPxUsZyqRKEnupI3qzkZZAIx6UoVvSxxiMJ7HlTWf7HzZsEAHhzc5fysxmLoL0ppyV/0fL3YtqJUle1CFE+kPK2cLRPkuVvcfKXSblQdmIdvtu6i8hYllRLSCH7OIz8g8XGdf31jsuEyiio0FtyeE4E07fF2YF4S9p9wh/XmMWcSV7dHjEUU4yeEfsUN/CJBKwjyaK05kBeYY13Fco8Mkc/2AR+FPGv99rifRHX3I0jf1EGWzB7An8tW/46u2FYyj6EkA8SQl4jhLiEkAXSvssJIcsJIW8SQk4Vtp/mb1tOCLlsIOc3COPGx5fjnOufGepu1ATsB9+tsPxVy/iJsCyCcY1ZdEjkz7g2XOWShmK3RRBNtI+8KUT+aSx/f7m+kiRbdfSW+azk+L0nY45fRI1hW3eJy0ZBe4LsE7H8PZKT/QsqbTmp345LuezDSLUo+AVE4p3e3ogfnXcInvzSiVwycdxgdS1VBE/GIsqZ0Dv2mujtt9UDngjZ8FFdU1ehgqZsPPmz7VzeSWP5xwz6zf7v1bYIDpnZzrf3+P4ZNgPRaf7DVfZZAuD9AJ4SNxJC5gM4D8D+AE4DcCMhxCaE2ABuAHA6gPkAPuQfa1ADrN7Ri02dhaHuRmp0Fcq46Sl1LRSmgIqLgjAkRflkfPKPWP5uVPapuBS2H78e6YEmGUo+1hJIIslI45q/wvLf1VfmA9Onjt8T1/qO69DnI5p/WG4RHb6MdJJkHwCwUxSQYaTPLP9CxcWuXu8ei7OAjEVwziG7o70px++fWHVTGbtvWZH8hFXXnYmpbQ28zaD/UjSOvy+ydKfiOruKFe6cFWWfIwWLnG1nJB8if1Hzt9JZ/kwGa8ra+PBRs3Ds3EmwSDTOX8fxw1L2oZS+QSlVZeecA+AOSmmRUroSwHIAR/r/llNKV1BKSwDu8I81qAH6yk6q+ibDBY+/uRXffGCpcl+c7JO0KIhNCMY1ZTkxMYiyT7Hi4LQfPoUVW3tgEzVpE+hkH+l8QvRHUkx2UNsnSv4dvSX+/bGMT4asIEWITmiRFFsl2adRI/uorinJ8geCctQs6ue5Fdtx8NUP469LNoWig8KWsvfadQOHr9Lyt9ULmQTtiJp/mLbY9cjkr7uiBsHhy3D5GfsGnyNhrT808AgzLzFkVyUxMbAcjIacjfamHH77iaNw4O7j0COFeuri/EdabZ/dAYgLaa7ztwHAWmn7UaoGCCEXAbgIAGbNmlWHLo4+FPya4yMFcTH87AFTOXyTBjhbY/k73OELbOksYukmz2fglWVWk7xa9lFb/paVrM96FTW9aB9Z5ujoK3OpxrZI6Luc2JzHps4CMpLDV3a0hix/5vCNkL+6X0lgmj8bBJ5Z7tXIv/QPr2BSa54fp5JoKq7L+9GikO1sjezD2wkRcHgfux2y7KMbiJlEI0o1ogTEru8IfzYQsfxDSV5+mzGWf3Pe+06bhHDQplwGvb7lnx+umj8h5FFCyBLFv7pa7JTSmyilCyilCyZPnlzPU40aFCrREgXDGXEkziwqleafFNJnWwTtPvmH1q8VQj13CHVlLIso4/mJpqSzzuGrm0GIYA7b5Vu6sXJbuADbJ29dhFV+zRlWBoJhYkuOb7dDMwJR88/CcSk6esuwiLhwjCz79Nfy98iLlcFmklxnoRIqJieHRgLA1u4iFq7agYxFeBSSfP5Y8pcSrVTXIzt8dQazmKmsar+tIYu/fPZY/Oi8Q/2+hVc9Cwb75GifWz52JBqz3vWKmcXN+Qx6il5tn2xCtE+9krwSLX9K6cn9aHc9gJnC+xn+NsRsNxgg+krx0SLDDXHkz+LGu/qh+dsWQVtjFhWXorfkcO1bTPLa0RsuKqZ67iyiTvKSNzHyF4lBB1bYTUxwEvH3t7ySFDYhoESw/FvyfLs4KIhW4b67eaUa7n5xHXKZYJ0BebBU9XG/aW0AvEiqp9/apuwbl338QWCNojgau0b59Xk3PYuO3jLyGStEguJ1sLDQxDZtmfy9v2llH7YOryj72H69o4pf++iA3ceF+ia+ZrWYRIdvXkH+TTkbJ+w9Gb9f6IkdYiJYc96rL0QIUmj+w9Phq8N9AM4jhOQJIXMAzAPwPIAXAMwjhMwhhOTgOYXvq1Mfxhz6ytFqjMMZuoGKUsrL96qSY3TLATJ4RcKC+vaq823vDlv+KgtfV9hNJk/2cNopyF+O05fBumhLmv+k5sDyFyGuOnXiPpNx4O7jsKuvzOvuA9H7pUpc229aG17+2nvwb0fMjO70waN9fPnnbWHtYREqzb/D97+UHVdJlBmL4Jr3HRBK5gq3GZ1N8Pf+OeSsZB1Uso84y5O/8lCcvxDe6Vn+4TZFnLiPp1iwwU4c9Nobs9jZW0I5TYbvcCR/Qsi/EELWAXgHgPsJIQ8BAKX0NQB3AXgdwF8BXEwpdSilFQCXAHgIwBsA7vKPNagBCopFOGqBXz+zErMvuz92Wt4fyJY/k2iKlWBh7f5q/vLKVkA4ykes/6/T/FOHejLyJ/qMYPFcqlBFlmTFpBQ5x2DG+EYAiph9oS1CCOb7FnwuY/PzyDMl3eDjRefoL4DJPczy18lvcVY6Ww1Lhm1ZGNeYxUXH75ncptRHtidi+WuuUyX7MF8Mey1C7G8ox0LQ/OVTPfz54/H9c718BuZgFjX/iS15dBUq6Ck5/Leqk3eGZagnpfQeSukMSmmeUjqVUnqqsO9aSulelNJ9KKUPCtsfoJTu7e+7diDnNwijzy9LLK9IpcKhVz+Mq+5LN+5+/5FlvP1aQrZIGbH1CNY+0/wffm0TfvXMSu9zKciflQeWi5oxiEXcLI1WL6byy9vD5/O3p7H8NccctafnXNzV5y9YYof7tLcv6aySpBZZq29r9CSufCZIRpKjo+JkhLg1iINon+QSFgxJ94O1GayMFp94Bagcvupz6M7M11+2xdlE0O+4+yPeHtuy+LMmP3JzJ7dw0mdyT4OimiibpXnnVZ9zWIZ6GgRwXIq1O9Qa6GCBLVGXRvnZ2VvGr/+xKlW7/Iddgx/hss1d+PRvF0UWzwaAw655BFs6C6E6MWzpvotuXYSv//l1f4Hv+AsUSwWEZB/BMBTJX2v5Q22lauP8NRnB4b5ZymPGNWaRsy1ej0i2/FnpZbnyqEz+LIaekIDcrv7L63hxzU5+TFzh0bgM5UD2ia9jI34/SYXuxOimOOgS2+I+m+zwDUtJQeZuun7YJDAo9pzcjE8Ksxbxd8MWehFlH7GeUpDhq5F9hiraxyAdfvjoMhz37ceHdABg5J9EjtWWgOBaeQ0UpS/9YTEeXLIJr23YFbHgOwsVPPLGZk74QHRFrjc2dipryYsQlykUj9VZ/jpr3CJBBcfw9vB7Me47jeWv6n9j1kZLQ4aXobYlJ/RsKdtXPjcDK+ZWrLihgeHLf1gs9D/G8o8hGp7kJWj2H3nnbOw5Odw3UX5JS+pJpbBFwpe/E/0pdLKPtz0XkswCv0Jcnz1r339tW6HVxS4/Yz/lZxoUA5y4jkBSbZ9hKfsYBHjiTS9KQwwhHEyIhcJki3rxuo6QFCTHv6dpG4hfaKRaUKgHqaxtoacYLLJdctxQ319YtTOF5W/xB1tcACVE/l3B96TLzCVE7WzThXp6cf6xXUPGInyQFtGYs3ltHnaceB5mpR46qz30OXkW0uZHzBTKToj8xWuPkzVEy/+mCw7Hdz94MH8f1PYJLNiDZozDV84MSO8Tx87hfgcgjdXqk3/CDEGur3Pjhw/D/tO982hlH82pGdmGi8UFMlms7CMl2LH7yr6rD/rLOopgPhKxWRa6CyQXhatXqKch/yqwvbuIC29+PrT8HANfiDnhi6wXxEgH8UF/ZW0H3nv9M6EKmJ1Vkj9rrhbZw+x3TKm6vZxtcc1/fFMOpYobGqyWbeoKfe7k/aZg0VfC0ciWBY3sIzh8JctfV95BHecvvw9kn6SwPNsiKJTVln+zkPyk8h8sveY0/P6T74htv82XfYplN0SY4p2OIxPRAj9sj/F8jVpAkH2E3/i4xmxowZWvnDU/4hSVP6M8r/CZfzl0d1x++r7a/TYhOOPAafj3o/eIvR7dZapkn+a8zduJuz+hBDsx8sff/O1/PQirrjszfD5+7aLlHyTFJcX5j7RQz1GJW/65Gk8u24rf/HN1ZB+LKhF/UKu392DTrsGptSOSv0hyW/yoFpFAq7b8uVOrlpFE6vrrWdtCb4mRfxaliosNHcE97ClVQp8jhGBiSz70wGUsIczRdbFiazeu/vPrITlCnKHJJRMYdA5f+WHklr/meBEZmyhDEhtzdqj6o62QnBqydmLVUGb5lxw31M/wovX6z4uhoznJwctkH7EPbY3Z0KAlQ1UZUwTri7j/B/92CD55wl6h40JJXqz2DifrKjV/vi5BcEBzLsPvV5zM4sX5B6+Z5DW+KeefU/9ZcVe7kNMwVFU961XeYcwh0KaDh+yE7zwBABFLoB7oC1n+wfbuokf0YtRLR2915K8qhdxfsAeAUihzErI24eGd7U05rNnRiw0dQUJUbylcv0g1exAt/1KF4uLfvYQ3NnZq+yQXS+N9RTiUUmw/9F7I+EwO9bRC1TAZGjI2LwMA6GcjSWDRPjLCUU8x/ROul8kjbQ1ZFMpFpfXe1pCNjf5hFrs84F1//qHoKVbwvYeXRc6rbieckAUgFVmrwDT/ULSP4GOJ1/xJ6PUVZ+6H98yfGkoKk8GMJrFVcXbEBiHdZRjZZ5iDWZUJvsi6IST7CE93t6+fiwtdiJb/q+t2adtcsn4XlqzfVVPZh6Hs6FdeYpZ/u2/5s0qle0xsQk+xgorrcsJR9SljWcj5WZybOvtiiR/QJ16lDfVkD2eqwm4WQUGYgezW1oCnLz0JlkXQIlS7TOM8VqFNUTETCDth42ZwWQX5s8zbvILk2xozsZY/Izn5azrroOn4tyNmpSJceT8bBywST5q6Qmk5heyTpj3xGNanfMbGcfPiy8/ocgE+f/LeAICZE5oibevOWUsYy78KcL1asa9USY606SyU0ZRi6t4f9JXUmawsTl6M9RbJ/+zr/46nLz2J/wBFnPWTv4feJ1RVSAV2D0uOqyzT4LgUPX6o5/imHEqOywe2yS15bvk35zIolEvK++2l6vtZqFu8mjMTm3PYrnHGe6St6CtJV9hNLO+QJPvYFgnlS1BQfu9bRMtfk3uQBFXFTADY0RP4OOIXTQlbw0AgJeXsaIjnuMZsLDmlqRmU5jhVkpeVMHAkh3qG5SN2fKLsQ4PyDmkQWP7h4//75Hm4+KS9gv7rLH8T6jk0eHbF9lDSkQ5M89dl2FJKcdBVD+PSuxcr9w8UfRqHL5N9RMdnpNplSou+FtnDzDL+8WNv4a+vbYrsL7teaQdCPNIpO5Rbre1NOa75N+XV5YoBz5JnDzjL5NWFSgJ6K5to4vblTUHd92QrLWMTFIRQW9Eibwk5fPtn8TUriqYBYcs7rjqGSn5hlr8ySilrxzpzZWK++SOhNZ84IdoJepmqsBu31DWf0W1nGjv7jTBfBjs+bgAXv5O0pHzIrPEAghXoRGRsK1G+MrLPIKKv5OCel9Zhe3cR5930LP7r9pcACA+9ggSZZa0jUiZx/PHF+tSxC5G/YFAzy7+isfyB9OH7NdH8/b+LVu/E2h3R4maO66Kn6KA5l+GkwmSgcY1Z9BYdVBzKSU5n+TPZh8Xz76GY2TDpSF4dK9RWGstfTPJKeKLEBUDmTmnB94RQSlE+yVhWvzI70xBSvOwTvQBWME4VyUYEqUtVsE2cSZx50DS8a9+p0ufZcVVo/pJGXr3mL5F/NhxtE9dcNau2MRwysx1Lvn4qTtl/t9jjBru8g5F9FLj6L6/j9ufX4PrzvZKur8To4gxJsfBJiUkDhSglOJRiS1cBb2/p4Zp/WaP5A8lVMhlqUTQu6Xdcdih6SxU05WyuzfYUHdgWQWtDxrf83WChEsX9toUMXxbttMdEz/Kf0JzjkT6TWvJYt7PPr+qZnvzlLdVV9QxI7KHPHR+ux18Dyx8AfvyhQ7HXZP1MJ24GpyK0z79nb+w/fRyOV1iuDHdcdLRSOhQHQ1XxM25tV6H5s8F2z0ktAIAj50zACqk8NpBcz5/NJvjMJZXsE7yuRo5RrWEgg82CcnZ4VbM6cf/Ytfwv/+NiPP7mFuW+9X50CbOamZ7Pvhz50fnF0yv4ax1Bsmigek3hZIfvB376D3zo588Gso8gL8g18tOOS4NRMJRp/s35DLc0u4sVZG2C5ryNvpJn+ccVFxPJf2tXEYQEhdHEEDtWJtnSJHmxtmREZB/R8k+h+e8xsUnZdrMU6tlf8n/vwdOx/3R99Ems5q8g6Kxt4cyDpsU6s4/ecyJ2b2+MtieGjsb4uqrR/Nl9O3DGODz1pZPw8WPnxH5WBivpnJNkH3E5Th3E7ySt5Z8WrDlZRhuWVT1HKiiluP35tfjor16I7Lv12dV4ZrlXz5xZlUyX1X0H37j/Df5a92Ax8h9ozO7LazuU03ZZ82eSCgvrFElS1m7Tav4Difb5x/Jt+MviDdoIDIaK46K36Fn+WW75V5C1LTTlMqi4XpQQGxjEwZatw2qRwKrb2l1EW0MW45s90hdLRDf7swddbR8gePD2m9aGE/ZWR3WEHL4pMnzv/vQ78aeLj4nsE6s+plkPOA1WfuuMiM4e9zXGFXYTcfen34m7EhLOgPDvnZGuCJKCcAFJ8xeOnTWxSTlgnXXQNPz03w/j7w+eEQyGsuzTwGUfhP6qIMp2tY6/Z/dCjqoy5R1qCJHEZCL96r1L+H5mLctri4ofkWvP6GQf1sZAsvUeX7oF77vhGdz23JrIvpDsI1wfi3ARo30KZQcHCnHJaR25A3H4nv+L53DJ715KPK7iUvSUKhHLP2dbIXJk1pF4rTd/9Ag89LnjQUhQz99xKdqbstxp2VOs4Bf/sQAXn7RXEJ8fE1nDpvZ7TGjCkXMmKI+pZg1f2yKY1JLHwTPbI/vEFa5IirbSgBAS0fHj5LukMgsMh+8xXns/wu0J5B9n+VcR5y9/VyoL/PrzD8NBM4J7/KdLjo30g52TL6YCpvnHR/vw1zUmZdacPEMy5F9DiFawbkUlIFg8PI7zOvrC4YO8Bpr0oVrIPqu2e7rmW5u7IvvECBKXBhYcc3iKPofekoOpbXn88kLPIqy15e+4FPe8tE5NMgnXX3Go7/C1+UPZW3KQy1ihSJacgvxb8hns4zsnRaJpb8px8u8tOTh5/lR86dR9ObFnbPViLkBQAZOCxiThBNZrmpW8dGjOhR2mtTIsZUKPc9zXWsoQyVKp+RN2XPraProM67QQDQNAiPbhfYkhf1H2STlLSgv225Ed6ynH4+rPV59mhzdES365ZjUiQL2QiAw5W5bPGgRLu7dUwfIt3nnSjOI7e0pKaSduECqUwrIP+4Fz2Ue45r6yg4aszQkvbRRPWsv/tudW4/N3voLfPR+doSSBWf5N+QzPfOSyjxAHP3dKC2ZOaMQVZ6orKYbIvzHLY9VFiOUB0pTTTUrCSRvnr0OT5BSslcUnSzlxY3itY8rttJZ/FZp/tZ+Vwe4HW/OXyT5ByKX+s+IyjkkDVrVg55VlLBPtU0OIYY/lmNWpOqX1Y9lXQAWX704pcYgRqZhR+4lbFuIfb28HkDxV7CyUceg1j+BTJ+yFy6TiVrwfijb6pNo+uYzFk6UAL36eoVBy0Ji1+UMjW+irt/fg/J8/FzlHWvLf5kfYyLXnASTGlXqaf9jy7y5WkJcs/5Z8Bk9f+i5tOyLhtTdllZmv7Lvwauyr2wmt4pTgFLZJcmRGnKTQFLH8a/PQywNObWs0xSNJ9kmb4Rs3KImf/fJp+3KHOuCtkNYo3Vd2/G7jPIf/8X6GLmsl7jsKrZ9cc9nHa0+XeVxrjFHyD4hZtNArkrbfKVn+Kotpp2T5u9xfEBzMiB9IJgdGnP/vybc5+RcrDnYl1ONRkb8I2fJvzNn8xytHzNz6z9U84klE6mhVv10Vx5QTwkq55Z/L8IzSnmIFLfmGiEM0vgsEWZug7FC0N2ZDZYjFY7y2ktdPpVRfLkCUfQbiBJTJn8QYlh88fIbymlSQyaSWZTqSkOjw5SWd+3/fRFlL9kV8UbEmMPuu505pxdOXnsQjwVgz6mJ+3m9ArORZe4ev91d+dusV6jkmyV+0gpmFvmT9Lnzv4TdDx8myD4uHF5+djl7J8mfkryG5pGm1OJi4LoVlEXzhzldw/6sbQ3XTZYjlHSiN/oDEQa6v7Fn+lsby1/3Y0lr+7BJ/9NhbeG1DJ35xYRBtkrQOcNlx0Vty0JwPLP+eoq/5h5Kgkp8Ib+EUqpR8vP3RPlsk/P2GtWb/rzQIiMv/DcRKk7Nz49r6jpAclgTVOrqDhbSa/0DkJrH4nlgcLw3E3IRAvoseR+BNWlWL09cK7Px5e3DIf0xq/qLUw0jxrJ/8HY/7C7IwdEmyD7OQRSs6YvlTpvlryD/hmxQHE5a4wsogyFFHIkIlnQXNn4H1x3UpCmUvSYr9eGXNX2cFx0WJbO0q4op7XkWh7ITI8dE3NoeO69WsA/z+Q3cHAGzaVYDjUkxta+DSTclxI9E+afRWNtAwyefWjx+JR79wPN/PvgvH1ZceVmn+8u0Ro30GwgdN+cFx+NYiWS8txFlHnOY/kEFTNATSJFPpwFrRlfrQ7asVWMtiHsLNH1nAHdK1xpgkf1GPrzguPneHOgQxavlHnblRy5+1GxwjPsRJPx5xMGHRRozI2SIgqiZk2Uf+wbD+sKigxqzN+/L5O1/Gz58KEtV0PYxzDH/hrpdx23NrsHDVsqRhuQAAIABJREFUzlhLRR5QGT64YCaaczbe9ge8mROaQrMXFufPkCbSgnEcK3Z23LzJmDulle8PKk7SUNiniDD5q88TivYZAGPXK8Qv6vBNJn9VqYb+QBywlZq//3cgfgjxO4qrMJoIPrjH+xdo+PCagUf72Ba++8GD8dj/nBAph1HT89Wt5WEMkbyLFRf3vrxBeZxIVJRS/jlx8Ngpk7//I9ZZ6TpuoJSis1AODSby6mCsDjyzrNds78VV970Gx6XhSpE0qvkzy58dJ1r+27pLuPaBIFFNx/5xBuNzK3b47Vqx0Zy6CCqml6/0o6/2mNAUynTMZsLRPtVoxDrZhz1sLqXaZKM0hbyqKe8QB5l0akUu8jUlkf8jnz8eT116Uk3OLV5TXALZQOYiokY/EMuf3Sbd2g5AWCqs9RxAdH7/6+EzlOUyaokxSf6iVb6ly6sV/5750RG2sy8gqkI5KEEsfn5nbxn77taK75/rabDMgtEtnqEjh9ufX4uDrno4tNwiI/8st/zDkslnb38Rv/7HKizd1Im+shPIOAqHL/NzsBlCQ9bWRivoHJs6ucBxKR/sihU3lrSKGs3ftrzr7Cx4FT13H98YshRzNkFTNr3DV4Suxj1rwrP8/XZjLH+dRRiUd9BHgDRmbZ5XkRa1s/wl2SeBaedNbcXk1nz8QVWAWf+qwnC1KG8uDspJy0XGthMT6sl9E/WUfVJmO9cKA7rzhJDvEEKWEkIWE0LuIYS0C/suJ4QsJ4S8SQg5Vdh+mr9tOSHksoGcv78QrXK2ROAsxSgrSim9pQonUDE6Znt3EZNa8jjML9saSEPVWf5PLfP8Day0hNgG+0HLa78yK9pbF9bhSUIOpZGHgPkp2ADSmLW1PzLd71sXJSLOfsoO7ZdD0SJBpMz0cY3IZ+yI7JOxLX5daTNRAX2N+5DmL8Tqh49RvxYhWv66ezdvagvevV91U/haEY08S6pFae5qwAZflexz0wWH42PHzMGcifpCdNVgIFnR7JPK9ZwREHO9QmW5ATISyB/AIwAOoJQeBGAZgMsBgBAyH8B5APYHcBqAGwkhNiHEBnADgNMBzAfwIf/YQYXosGUhjWJssAq9JQcOk30cF6f84Emv7HNPCRNbciGrG4ghf80XyzJQRc1/XUcfdvSU+HSZ6fXst8mIvFD2FjxhU15Kw/pxa0OGS1bM4RpL/pp7oC1dIVjzpYqbGNGjgliMjRUIyymchUzTrcryr0L2icpJ3nsKPRmHCrtVeU/jUDOH7xCGegLB0pKqaJ89J7fga2fPr3lyWX+QKsPXL88BxDuw+3V+fo7BEWQGdBZK6cOUUqaNPAtghv/6HAB3UEqLlNKVAJYDONL/t5xSuoJSWgJwh39s3UEpBaUUa3f04u4X1/HtbH3YJH2tp1Th4ZvLt3Zj2eZuXHHPEmzrKmJicz7kPATU1SYB/Q9rXFNAUGwg+Oq9S3DYNY9oZR8mofSWKugrOzxDVJZ9xjflIpp/U87WFiHTGU86i6cUip5yY6OSdBAtf2apZ4VrYNfTJBRjSwu95e/9pYLsI5MQ4cfEOHxZtE+KDN9qUIvaPkD0Xg2y4c8t/1qXQ6g1YjN8hYHhhg8fhm+9/8DYBYL6A74o0CDdplrG+X8MwJ3+693hDQYM6/xtALBW2n6UqjFCyEUALgKAWbNmDbhz77vhGdgWwZodfaHM0427/PVhBfK/4oz98NzKHaEwRU/z956aJeu9NWH33a0VL67pwKTWHH/oGe/pMod11mObQFBT2/KhmvtZOdqH98kj8r6Sg76Sg93GeRazbKGPb8pyJzXX/HN21bKCjtNFsldZ/mlCC22LcHJoYBqxyvL3I35UC6vroCP/0w/cDb9ftA6HzByPx5d6sptM3uK7pESwuNLQQ4lonP9gW/4e+ZfjlhAbBLzytVNiI9bYXVKGevp/beJZ/h86cuCcFDmHRnqsFxLJnxDyKADVEjRXUEr/5B9zBYAKgNtq1TFK6U0AbgKABQsWDPhXo1uQxXEpWvMZTGjO8W3HzJ2EpZvCxdMqjhty4gKBBDGpOc+tP/bjKmsILw3fzpnUjGWbg5pDzOqVSzEXuOXvoFB2+RqwlFKhaJWFhqzNH7xUmr9GpEgl+zhuxKmbZiZgW4RbqKxWf05y+ALgqfrVWP66OOl37TsVK791hpecJVjvOiTV/0lT1TMJt378yEjuyECRlQu7Dbbs4w++nX21va5qIc6uVbBiyLcWyWhJYC3XurieDonkTyk9OW4/IeQjAM4C8G4a6ALrAcwUDpvhb0PM9iHD5NZ8KFU+n7UiqfZlh4ZCPAFgS6c3g5jYElj+QXkHNeHpvtaSMLAcNKMdD70WzDoYCTLJhv0Q2UPcXayg5Lg8Dt5xg4iOFr80MssZ6CtH4/wjfaxW9pEs/2JFkqfKacnfu87GnF9f3QpKNASaPyP/2uiicvJOPPlrtgsJYtrPpxwUjpunXjNgIJAJa7At/9MO2A33vrwBe09tTT54CMF+UmrVJxjg64W0dY5qhYFG+5wG4FIA76WU9gq77gNwHiEkTwiZA2AegOcBvABgHiFkDiEkB88pfN9A+pAGslYuo7UhE4qOydlWtDaOG7X8N3V6ktHElnzE4SsPFAw6o0t0EB88I1zrnZc5KAWhpyIRs2UJebSPS/kD3tKQQcYivO+9ijh/GTrLMGmhGnYdsuwjDwYq2CSQfcQEIzbwZbnm7w1w9Sqn2y/LXxhAhqPsI4NFpg0WTjtgGl696hQcOEO/ulgtIM7e+4O4hYYGw/JnhmM9w0lFDFTzvx5AHsAjvgX1LKX0U5TS1wghdwF4HZ4cdDGl1AEAQsglAB4CYAO4mVL62gD7kIgdUuVNGfmsHZqu57NWJFSy7LgRJy7T5Sc25yIO35JG32SDwktrdmLG+CYeTy1GIIkLrQBBckyPsAqVWLFze7dP/r4M5VKB/PMZZGwLZcfFI69vxlu+nNSQtXnSWLSP/Sd/leavK+kgQrSaxVlYe1MOPaU+PgiwAS6NdXT9+YcmfvcMyWF2+nr+QWmIwdNr+4vvffBgnHHgtEE/b6sm16JWePrSk7T5HGnBnfuqff7fen697PkaNrJPHCilc2P2XQvgWsX2BwA8MJDzpkWx4uBLv1+MY2MWngaiiSH5jB3RicsO1ZLfuKZsKGwQiFYIZWAW+L/c+A9Mbcvjuf89mbcPACfsPZmHxjEEyxl6JOq44VLSLCuYSVUe+Xv7WvIZ5GwLFZfiP3+zkH+mKWdrw1F123VyQVK0T3cxnNX7sWPmYOOuPjy4ZBPfZlmE91kk/3GNWazv6AuiffwBLs0DctZB0xOP4ecXwjWTjtFtj8vwHS5DwgG7j4uUOB4pmD+tTTujrkU2LDMAVfIm3zfgs+jBjK5h4/AdydjSWcQLq3bgvlfU5RsashYKZTdC9PmMQvZxqJYUc7bFw+d4tI+O/IUBZHNnEHVUclxMasnjlo8dGfkMkzwYibqUhqKB2GtGmo5L+RTy4pPm4u4X14UIOuPH1OuITpa3GHSSVVK0T49E/vtOa8XcKS0h8rcJ4QOmTP5AMPupxvKvBrpIC3FGmGj5jwDZp8ah6YOKB/77uLq2z746lY1z44cPw8+eehstufpRJjOu6ulXEDGCfwrJmDmhCTddEKTUz5vSEtrPtOUGacHknB2VfSquq7X8s7bFnUVbugqYfdn9+JOmXpBIzCLKFZdHtMiQk2Mqrhuyphn5sz4z2eeI2eNx/N6TkbEs9Ar+AnbdOgtDZ12lkn0cGiX/Upj8c7YV0ewtKxh0RM2fzYLY7Idr/jVOhOGVOWOa1Vv+wf7B0mv7i+Hev6GEWO5DxvF7T8Ztnzi6rlY5s6FGhMN3JGD/6W389R5SCjkjEtnytywSsfxLFVcZvsm0ajZaszIN4gIuIiqaJKiy44aSmuLguDRUIC0gf1/2ccNZq1mbhI5vSLCe9ZZ/Ss3fkS3/sOY/d0pLpMiXTQhPomPRPoCQIOT3lUX71KuWus7qikvyIoTgo8fMxon7TB725DpYxDISYXHZZ2jOz2qHGdmnRhBv5EQpGoBZ/PlslHSjlj/lX44IRmLsoXp7q1eSuCWfiWjdrB01+VNturhscVcciu4ik3osvtwkux6HUrhuELqWta1Q7D2zrLWyj8bCF2csBb+QXNa2QtdTdtxIaGevZPnvP72NL0bPkLEsLpU1ZETLP1h4HQAac+k1/2qQJswuLob/yrP3B6CfHQ2XMWG4D05DCRJj+Q8GuOVvZJ/a4c6LjsYXT9k7Ys0zy79BkQSkWgZRZREzwiYkXNRLlpJ4O25UFgE83VxH/rL/QLT8p7Q2BJa/T+o7ekoouy4nMlliYY5hnXKi81eIl3/uz/6J7/orn7HrachaSsu/W7D8D9i9DYSQiGwjyj4Nuajmzwa4A3cfh3lTWmpaddKDOtSTnX96e2Mq4tTODgbWuZrBWP5xqL9TNw4sidLIPjXEUXtOxCXvmhchwXjLX5XkFf1ZiHq8OGLLFTgZHJcqyxqXHb3mL+cpOFQk/zw/F3OUXvfgUry0poOTlTyosON0urlW9hGuf0NHH970s6AZ2bfkM8o4f+bwvers+bj70+/0+yTJPhbhEU8hzd/PDmUD3CEz2/HIF04Y2KIdCuhK9h45ZwJuOP8wXHHmfqmcubWqx1MvGPLXI6j1NDTnZ8+XIf864Ji9wiGffM1M38o/88BpaPdTwOUBoey4ofBNNjMQiVWUmHSJZY7G8i/HWP5yWYeKb/lnbYL2pkDKkqUqXaVK7vDV/Mb0CWrhRXBYtBK7nuZ8RlnegZH/pNY8H1TlSpMWIYHsI5D/yf46Cx84bAYGA6oH78yDpqEhJiM6DYbLoGBkHz0CzX+IZB9j+dcPJ8+fise/eCJ/z8iMEdINHz4ML3/tFG9bJMImbPnzEMRM8EWJ35k8S/jP4+bgk8fvCSCqgQNAuaLX/OVZhONr/q0N2VAZigZp6T1mXMvtsjhvHSHpCnCJtX3KjostfoZzyV/ApTFr+6GeDj581Cw88F9eaB7zfYgzjawVtfzZ4Cpa/jPGN2HVdWfi0DpnpbLexGv+de3CoMBY/nowg2+ovmdnkC3/Ue/wlTFtXAN/zW62Sp9XW/5h8t/aVQwRa5yjpr0px39UqozXkuOiLafOUJRnERWXortQQUs+E+p7QyZqTQNR/0XS+qxay18Y0EoVF4VyiWv8rCRGseKiUHbRnM/wuvzsekWpR45o8KJ99N/HYKE/5R1GEgbLmTgSceXZ+2NKax4nV7noTq3Ayd84fOsDURphXK6q+pizw0sGyuUd2n3LP6eRfaLtWby6okj+n739Jb/tKjR/10WXT/5i3/MSqTPLXi5pLM8QZGgtf744vcsTvpZs2IUXVu1ALmMha1t4ctlWlBwXFiF8AOWWv3Cv5Cu1NJb/YIFH+/QjwzdV+/383CUnzcVPPnRov88rY5DWCRmRmNCcwxVnzq/J0pL9wWBb/mPupyBKHcySVa37KVr+rDCaGOrJZB/Rqpa/NFFrz9hB7Zo+Qfb58ysb8PqGzupkHwp0FSuRgnTydbDm5EWtcxn1j2v2ZfdjQ0eftjQFk8nEweEr9yzBS2s6vKxoof8VfzYAAG/7i7KLUo9KcuLlHYag/ADRRPuIGArF5Iun7oOzD05fpiIJRvYZvjDkP4gIZJ8o2chVPssODck+bY3RdUlly1C0YDNCVqss+3QVKig7rtbiKFTUln9rQyY0SMnXYXHLPywnxfmz/vH2dmVUEyFC0TrBofvmZi/ix7YIT1KbNaEJnzpxLz4TWuHnPmRC90rfB1Xo7WChv3H+Iq48ez7+dPExtepSTTEapKvRCmZIDpbsOeY0fxGBwzd6s0WLPmMTr6SzSzF/WhumtzcESw0KUo2cHNLakEGXL3nkbALGmTL57+or+3H+6WSfCnf4toZlH43mL1v+cSiUHaXsYxPCB0sxjp9t6y06fCZ13pEz+TqnV549H1//8+teGwmW/6wJTVizo1d7H+qJNCV70xpkHz1mTg16VB8MVsVIg+rxuZPnIZ+x8P5Bimwzlj+S4/xZSeSK6+KEfSbjFxceEdSZF6xZObtTtHQzlsUfPDnaZ2tXwY/zTyn7+KGenuYfY/lLa+IyxFn+hbKjlH3EqpuqDOWuYgVdfiLWpOYgAetfDw9+yCGHr4KD/vCpd+C3Hz9qSMMi48hxIGn3w8XgNrLP8EVrQxaXnrZvzReG12FMW/4sdFEl++RUso9LOTkwYs/Fkb9FeOXQbMbyVjZA1PLf0lWMLe8go+xSdPaV0dYok79s+Xt/dWvYqlCsRNctYG2pZB8RLBFrUmuQeyDOOsRQTxXBT2lrwJS2hsj2wQDrTZwsMhpoc7jkGxgMPcY0+cc6fCXZp1hxQWlAYMyKjbP8bYtgXGMWhXIRWYuA2szhGyb/rV1FlCv6JC8ZXX1luNRzOosRPnLUkq2RfWhMAnux7CjLO4iyj678Q6efdcwkHyBMNqLlv+9urdhvWhtmT2zCho4+bX8GC6yfcdxoiNNgNGFMk78jJXmJCEXqWARbu7yEpnGN4ZLCtkBoOvLf3OnlA7DzqSz/kuOGEsbOOWS6tiz0dn8hl3GN2XCegaYWfUu1lr9C87csQfP3Lf+MRUKzBLZA98QWdd0dUQZryNp4sM712fuDuKX82O3tzyAQ166BwVBgTGr+/3ncHHz82DlgkZsq73rYYrWwaZdH/pNbPVmCRe6IMoFDZc2fBCWJbcIHFLm+vSf7hDX/H5x7CJ4QspFFMHmlrSGrHLgYGFnJx8QtEl4oO8okL9siPO2dlW+Y1h6WaNhAIFdPZRgJzsY4XjeRMgajCWOS/K84cz6+etb8YM3MBLkla1vY4JP/pJacv82XCYTjZEeqbVk8H6BUcflsQZZ9tncX4dJonaCmfHzI47jGrFKyCs4fJavFV50SGzderLgoOxTv3ncKvvkvBwb9IYQPbkz2mTk+vHTeZ07cC4A+iWywHFkDQRy9G/I3GE0Y/k9jHcHILMkizdiESx2T/FLCaVaSyliE5wN0FipcIpJlHybjyOSYt+PJv60xq4xUYlDJE0mLXLNon9mTmvH+w3bn2y1CeIYvuxfi4jjTxzXg0tP2xarrzoy0qSstPZxASPhv3DH9O8EAPlsDXHX2fBwys31oO2EwrDCmNf9PHDsH33pwKbfOdcgKRM+cmdzyj3mobYvghL0n456X1mPWhCYU/WQt2fJnMxA5vl2OOJJDLMc1ZpV1ghj6o7IUyt6KZaJM5V1LUO2QWf7vP2x3nLD3ZBw1Z0LsKmTjm7LY1l3S7h8OYJp8nDY/EPIf6mHvI8fMwUeGcf6BweBjTJP/J0/YC588Ya/E45gjNpexeH15JhXFPdQZi+B9h+6OBbPHY8b4Jvzj7W0Aopo/g1yATbSUm/M2Sr1h8m9rzGJHj55U+xPTXax4ln/WskKft0jU4dvakMERsycktnnbJ47G7c+v0foChhOMsmMwVjCmyT8J17zvAExpzeN3z60BAExuyUdq5MdFfjDynOFr40zW0VnrsuwjFhlrzmews7fM3xMCtOYzsbKPqFHfcP5hylLSMnqKDlzqDTzs2i44eg/8bekWLpOxGYguKU3GPru14qr37p/q2KFCIPsY9jcYGxiQ5k8IuYYQspgQ8jIh5GFCyHR/OyGE/JgQstzff5jwmQsJIW/5/y4c6AXUExccvQdO3X83LscwZy8QEGuS5S+C1d7f2au21mXyt6xgaUg5Vr81n4FlkdhoH5HHzjxoGj64YGZMbz2wSCLWl7e/eQauPmd/P9rHO4ZZ/iPBgVst6sX9ZkwxGG4Y6NP7HUrpQZTSQwD8BcDX/O2nA5jn/7sIwE8BgBAyAcCVAI4CcCSAKwkh9V2lowZgJCeuG8sDexQPNbOIbckpzJytHYIFLy7Gspsiu5UNIDL5j2MrjsVF+/SDcVihNuYHsS1vBmARRGr7xJ17pMFws8FYw4CeXkppp/C2GQEnngPgN9TDswDaCSHTAJwK4BFK6Q5K6U4AjwA4bSB9GAwwfX/auEa+jTk/VeF/LG9AtvxVZRbGC8swzhjfGNnP2pfXrG1v9D4XR8D9rUWTsQhOO2C3SFubOguYfdn9ePLNrQBGl+UfrOE7tP0wMBgsDPjpJYRcSwhZC+DDCCz/3QGsFQ5b52/TbVe1exEhZCEhZOHWrVsH2s0BgRU6E5Oa2Cin4goW525L0TuqRcfHNweRRnLSFBAMIM1SzP/UNm8WIjuJRfRXajj74OmhEg2AN4t4ZW0HAODh1zcnnnukol6ZuCbD12C4IfHpJYQ8SghZovh3DgBQSq+glM4EcBuAS2rVMUrpTZTSBZTSBZMn6zNSBwPbur2FysUlIBn7qwiWrZErW/5Z24qsUjU+tAB7VL9n1rttWfjAYTOw//Q2AMBUXyKK0/yrlX2yNsGLX30Pvn/uwdG2LBJZmH00kb9x9BqMNSQ+vZTSkymlByj+/Uk69DYAH/Bfrwcgehdn+Nt024c1WIx6SPbx2V9l0TGCV4VaytJPU8KqVWwAsQjwvXMPxjFzJwEI/ANxte+rzUglhGBCc05JhKptI6FcQ1oEVT1r1+YvL1yAU+YPzXqwBgZJGGi0zzzh7TkAlvqv7wPwH37Uz9EAdlFKNwJ4CMAphJDxvqP3FH/bsMa2rqjlT2Msfyb7qMiRkX97U+BQjYPNyd/7y2YhU3zZJ85irZbI4g5Xyfuj0VqOu6a4dRBUePd+U/Gfx+8JAGhrNFHVBsMLA/1FXkcI2QeAC2A1gE/52x8AcAaA5QB6AXwUACilOwgh1wB4wT/uakrpjgH2oe5gq3FNbVNo/kryV0f7AEBTzrvl45ty6OgtwyIE93zmnaG2Rdg8n8B7z2YhYuSRDtU6fGOzlUch0YdQxeVVcycW7DEeV5yxX2hRGwOD4YABkT+l9AOa7RTAxZp9NwO4eSDnHWzcedHR+NvSLaGCZcfv7fkhPnTkrMjxjTGWPyuNMGN8I1Zu68GkljwOnaWPdmWkyy1/fxYiO2RVqFr2iS1t4O1jSy2OVtR6jCOEcOvfwGA4wcxFU+CoPSfiqD0nhrbt3t6oLGIGBI5QlaTDEqT+7YiZOHm/qTg3IfGKRQyxQWDe1Ba8vrET09sD/8PMCY043i/T/KVT98F3HnpTe/44xB3O2po7pWVUkn+a2j7zprbg0Fnt+MqZ8werWwYGdYMh/zqAVfxUWf4sYmZySx5nHaQvrczALX9fQfrW+w/Ex4+dE7L8n770Xfz1xSfNxfMrd+DJZVurtmJjS1X4+yanmHGMRKSp6pnP2LjnM8cMTocMDOoMQ/51AA/PVETisOxY3WpXMgLN3/vblMvgoBnxpXmDhKVqZR89WEXS6e2NuOmCw7F259AvvVgPjHLPhoEBhyH/OoBxvsryH9eYxdauYuoKl7YQ6pkW7NCqnbQxh6/2pZ65U1pwyv676Q8coWCXPtr92gYGDIb86wAxMUvGzRcegceWbsb41OTvtVGNFc+LzlVJZHHnYPWI5k5pqa7REYbRGL5qYKDC6EnRHAb45PF7Yt/dWvl7leU/a2ITPlrFohosvr4a8u+37JPi8NmTmpIPGoHgmv/QdsPAYNBgyL+GuPyM/fDXzx3P39di2cL+WP6MwtJE+zz6heN5uGqaM8SVkxjJ4FE+hv0NxgiM7FMP+BlgtSh/YHMrPv1nqqlQOXdKK07cZzJuf35NrORx78XHoLeYvBjMSIcpwGYwVmDIv45Qaf7Vt8FCPauQfdjflLOFIJFMf8xoX/zblHQ2GGswsk8dwEo/1MTyl8o7pAE7Nm2SVzBGjV3mG7tXbjBWYci/jujPAuq6NvoT7ZP29P2NDhpVMPfAYIzBkH8dwFb5qo3lzxy+6T9T7WLkfHZRVc9GF07wazWdsPeUIe6JgcHgwGj+dURNLP9+hm16n61W8x+79H/4HuOx6roz+TrFBgajHcbyrwMYfQyV7MMiVtL6m4mRPDiMw9dgrMCQfx1RC0ua9Mcqr3K2YGSfACbD12CswJB/HVDtik9xsEj4b7rPVEdgLIvYEJ+BwdiBIf86ohZcGkg41cf5p4WJ9jEwGHsw5F8H1NJlaHGrPP1n2LFpZyCG/A0Mxh4M+dcBtIa6D7f8q3L4enBT9iPQ/A37GxiMFRjyH+5g2bpVVfX0jjWWf/+x1+Tmoe6CgUFdYeL86wDGubVwoPZnkRF2aNr5h9WPstGjGX/57LGYMb4x+UADgxEMQ/7DHNYAQj3Tyk9sVmGo38MBu48b6i4YGNQdRvapB3zOrQWZ9qfaJNPu01r+xuA3MBh7qAn5E0L+hxBCCSGT/PeEEPJjQshyQshiQshhwrEXEkLe8v9dWIvzD1fUglS55V9NqGeVug+bIJhBwMBg7GDAsg8hZCaAUwCsETafDmCe/+8oAD8FcBQhZAKAKwEsgEdNi8j/b+/eY+SqyzCOfx96FdvQlktpui1ttQbqrTYrKQGRANJSL6tJY5oYIWpsVEw0xtTWJgQ1/iGJoiZGglq5eAFFjQ3RYLE1/iMtt94QCqvUwKawaqVqTCrS1z/Ob3aHyey2szvn0jnPJ9n0nN+Z3fPsu7Nvz/nN2TPS9oj4x2RzVEl08WLPTu/ND6NnCaea48RI83f3N6uLbhz53wJs4pXHmQPAnZF5EJgjaQGwBtgREUdTw98BrO1ChkqZmd7qcGoX3sxlQlf70NnVPi27MrMamNSRv6QBYCgi9rUcNS4Enm1afy6NjTXe7mtvBDYCLF68eDIxC3fju1dw/lkzufqiyd8eePQ6/w4+Z+TI/9Q0zhB84G9WHydt/pIeAM5vs2kr8HmyKZ+ui4jbgNsA+vv7T6v77M45czqb1l7Yla/VaPoT+QvfU/0jr8bDfKmnWX2ctPlHxNXtxiW9EVgKNI76+4BHJV0MDAGLmh7el8aGgCtaxn9yJmZiAAAGd0lEQVQ3gdy10emtGtJndfQ5r5s/m/f39/HRty3rKJuZnb4mPCkdEQci4ryIWBIRS8imcFZFxPPAduC6dNXPauBYRBwB7geukTRX0lyys4b7J/9t9K7G0Xgnvb/TaZ8pZ4ib17+Z5fNnd5TNzE5fef2R16+AdcAg8B/gQwARcVTSl4CH0uO+GBFHc8rQEzqdwgH48KVL2fXkMGtePz+nVGZ2uuta809H/43lAG4Y43HbgG3d2m/v6/zKndeeN4s/bLkqpzxm1gv8F74V1+kUjpnZqXDzr7iR62+6+fZgZlZ7bv4V13jB94R7v5l1kZt/xY1e6unub2bd4+ZfcT7yN7M8uPmbmdWQm3/F+WofM8uDm3/Fjd6h0+3fzLrHzb/izpjQvX3MzMbn5l9xE7m9g5nZybj5V9xEbuxmZnYybv5V52kfM8uBm3/FzRh5S0i/0YqZdU9et3S2LvnY25dx/KWX+eAlF5Qdxcx6iJt/xZ05fSpb1l1Udgwz6zGe9jEzqyE3fzOzGnLzNzOrITd/M7MacvM3M6shN38zsxpy8zczqyE3fzOzGtLpcJ94SX8F/jKJL3EO8Lcuxekm5+qMc3Wmqrmgutl6LdcFEXFuuw2nRfOfLEkPR0R/2TlaOVdnnKszVc0F1c1Wp1ye9jEzqyE3fzOzGqpL87+t7ABjcK7OOFdnqpoLqputNrlqMedvZmavVJcjfzMza+Lmb2ZWQz3d/CWtlXRI0qCkzSVnOSzpgKS9kh5OY/Mk7ZD0dPp3bkFZtkkalnSwaaxtFmW+mWq4X9KqgnPdJGko1W2vpHVN27akXIckrckx1yJJuyT9UdLjkj6Vxkut2Ti5Sq2ZpJmS9kjal3J9IY0vlbQ77f8eSdPT+Iy0Ppi2Lyk41+2Snmmq18o0XthzP+1viqTHJN2X1vOtV0T05AcwBfgTsAyYDuwDVpSY5zBwTsvYzcDmtLwZ+EpBWS4HVgEHT5YFWAf8muyt5FcDuwvOdRPw2TaPXZF+pjOApelnPSWnXAuAVWl5NvBU2n+pNRsnV6k1S9/3rLQ8Ddid6vATYEMavxX4eFr+BHBrWt4A3JNTvcbKdTuwvs3jC3vup/19BvgRcF9az7VevXzkfzEwGBF/joj/AncDAyVnajUA3JGW7wDeW8ROI+L3wNFTzDIA3BmZB4E5khYUmGssA8DdEXE8Ip4BBsl+5nnkOhIRj6blfwFPAAspuWbj5BpLITVL3/e/0+q09BHAlcC9aby1Xo063gtcJUkF5hpLYc99SX3AO4HvpnWRc716ufkvBJ5tWn+O8X8x8hbAbyQ9ImljGpsfEUfS8vPA/HKijZulCnX8ZDrt3tY0NVZKrnSK/Rayo8bK1KwlF5RcszSFsRcYBnaQnWW8GBH/a7PvkVxp+zHg7CJyRUSjXl9O9bpF0ozWXG0yd9vXgU3AibR+NjnXq5ebf9VcFhGrgGuBGyRd3rwxsnO4Slx3W6UswLeB1wArgSPAV8sKImkW8DPg0xHxz+ZtZdasTa7SaxYRL0fESqCP7OziwqIztNOaS9IbgC1k+d4KzAM+V2QmSe8ChiPikSL328vNfwhY1LTel8ZKERFD6d9h4BdkvxAvNE4j07/DZeUbJ0updYyIF9Iv7AngO4xOUxSaS9I0sgb7w4j4eRouvWbtclWlZinLi8Au4BKyaZOpbfY9kittPwv4e0G51qbps4iI48D3Kb5elwLvkXSYbHr6SuAb5FyvXm7+DwHL0yvm08leGNleRhBJr5Y0u7EMXAMcTHmuTw+7HvhlGfmSsbJsB65LVz6sBo41TXXkrmWO9X1kdWvk2pCufFgKLAf25JRBwPeAJyLia02bSq3ZWLnKrpmkcyXNScuvAt5B9nrELmB9elhrvRp1XA/sTGdSReR6suk/cJHNqzfXK/efY0RsiYi+iFhC1qd2RsQHyLte3Xy1umofZK/WP0U237i1xBzLyK6y2Ac83shCNk/3W+Bp4AFgXkF5fkw2HfAS2VziR8bKQnalw7dSDQ8A/QXnuivtd3960i9oevzWlOsQcG2OuS4jm9LZD+xNH+vKrtk4uUqtGfAm4LG0/4PAjU2/B3vIXmj+KTAjjc9M64Np+7KCc+1M9ToI/IDRK4IKe+43ZbyC0at9cq2Xb+9gZlZDvTztY2ZmY3DzNzOrITd/M7MacvM3M6shN38zsxpy8zczqyE3fzOzGvo/oCXxo3Z1n/cAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mV5jj4dThz0Y" + }, + "source": [ + "In addition, `avg_final_reward` represents average final rewards of episodes. To be specific, final rewards is the last reward received in one episode, indicating whether the craft lands successfully or not.\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "txDZ5vlGWz5w", + "outputId": "c7c5e9ca-6329-4ee2-f3d6-a1ba46b5aea2" + }, + "source": [ + "plt.plot(avg_final_rewards)\n", + "plt.title(\"Final Rewards\")\n", + "plt.show()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEICAYAAAC3Y/QeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO29e7wkVXUv/l3Vfc4MM+MwPIbnMDwHFVABJ6jRgA9UMA+MUX/k3sRcrwlqNNFcE6PxJj/NveZnkqsxRhN/JOIjapT4gkQTBUUQlccAIzA8h+cwDDADzHvmnNNd6/5RtavW3rV3dXV31TmnT6/v5zOf0121a9eq6qm1V33Xd69NzAyFQqFQjBeiuTZAoVAoFLMPdf4KhUIxhlDnr1AoFGMIdf4KhUIxhlDnr1AoFGMIdf4KhUIxhlDnrxhJENFuIjqhhn4+SERfrMOmuQIR/ZCIfnuu7VCMFtT5K+Y1iOhBItqXOnvz7yhmXsbM9zd87pcSUZyecxcR3U1Eb27ynArFbEGdv2IU8Mupszf/Hp3Fcz/KzMsALAfwBwD+kYieOYvnz0AJ9JlV1AL9j6QYSRARE9FJ6efPEdGniOjbaYR+PRGdKNr+LRFtIqKdRHQTEf1Cv+fjBN8B8BSA56b9RkT0PiK6j4ieJKJLiejgdN/nieg96eejU3vfkX4/kYieSo8/iIj+nYi2EtHT6edVwvYfEtGHiejHAPYCOIGIXklEdxHRDiL6JAAS7U8ioqvTfduI6KsD3F7FGECdv2Kh4EIAHwJwEICNAD4s9t0I4HQABwP4MoB/JaLF/XSeOupfAXBo2j8A/B6A1wI4B8BRAJ4G8Kl039UAXpp+PgfA/QDOFt9/xMwxkmfwswCOBbAawD4An3RO/5sALgLwDAA7AHwDwP9MbbkPwItF2/8F4HvpfVgF4O/6uU7F+ECdv2IU8C0i2p7++1agzTeZ+QZm7gD4EhJnDwBg5i8y85PM3GHmjwJYBKAqdXMUEW1H4pS/CeB/MPMt6b63AfgAMz/CzFMAPgjg9UTURuL8X5LSNGcD+CvkTvqcdD9Su77OzHuZeReSQescx4bPMfOG9NrOB7CBmb/GzDMAPg7gMdF2BslAchQz72fmaytep2LMoM5fMQp4LTOvSP+9NtBGOsC9AJaZL0T0h0R0Z0qFbAdwIJKouQoeZeYVSDj/TwB4udh3LIBvmoEJwJ0AugAOZ+b7AOxBMgj9AoB/B/Bomi/InD8RLSGi/5+IHiKinQCuAbCCiFriPJvE56Pkd04qM8r970VCA91ARBuI6L9XvE7FmEGdv2JBI+X33wvgjQAOSh35DgievArSyP6PATyHiMwAtAnA+WJgWsHMi5l5c7r/agCvBzCZbrsawG8hoWTWp23eg+Qt5AXMvBw5NSTtk6V3twA4Rlwfye/M/Bgz/w4zHwXgrQD+3uRGFAoJdf6KhY5nAOgA2AqgTUR/hiSK7xvMPA3gowD+LN30aQAfJqJjAYCIVhLRBeKQqwG8E0k0DwA/TL9fy8xdYd8+ANvTZPH/28OMbwM4lYhel9JLvw/gCLOTiN4gEsZPIxk44n6vVbHwoc5fsdDxXQD/CeAeAA8B2A+bJukXlwBYTUS/DOBvAVwO4HtEtAvAdQBeINpejcS5G+d/LYAl4juQcPYHANiWHv+fZSdn5m0A3gDgIwCeBLAGwI9Fk58DcD0R7U5te1fT8yEUownSxVwUCoVi/KCRv0KhUIwh1PkrFArFGEKdv0KhUIwh1PkrFArFGKI91wZUwaGHHsrHHXfcXJuhUCgUI4WbbrppGzOv9O0bCed/3HHHYd26dXNthkKhUIwUiOih0D6lfRQKhWIMoc5foVAoxhDq/BUKhWIMoc5foVAoxhDq/BUKhWIM0ajzJ6JjiOgqIrojrS3+rnT7wUR0BRHdm/49qEk7FAqFQmGj6ci/A+A9zHwKgBcCeAcRnQLgfQC+z8xrAHw//a5QKBSKWUKjzp+ZtzDzzennXUhWOjoawAUAPp82+zySdVAVCoUHl63fjF37ZwrbuzHj0hs3odMdvFz/tt1T+M/btwxj3rzF5u37cNVdT/Rsd//W3fjJfduy7zc99BTu3LLT2/bux3Zh+97p7Pu3btmMPVOd4Y2dA8wa509ExwE4A8D1SJa5M//jHgNwuKf9RUS0jojWbd26dbbMVCjmFe7cshPv+sp6vPdrtxb2ffmGh/Her9+Kz/80OI+nJ9782Rvxti/ejB37ioPLqOP8j1+DN3/uxp7tXv7Rq/Ff/vH67Puv/cNPcf7f/sjb9jc+cz3+6UcPAABuefhpvPur6/Gn37q9HoNnGbPi/IloGYCvA3g3M1tDaroGaWFRAWa+mJnXMvPalSu9s5MVigWPfTPJgl9bduwv7Nu+J4lAn94zXdhXFZue3gsgeYtYaNi5v/6IfO9UB3umk373TCW/zeO7ir/NKKBx509EE0gc/5eY+Rvp5seJ6Mh0/5EAer+bKRRjjDLXzKV7y9HXQsYKxAyY9a+Gue/zAU2rfQjAZwDcycwfE7suR7KQNdK/lzVph0IxqihzzqSee9bBYLirH9KIDqFNF3Z7MYDfBHAbEa1Pt/0JkvVHLyWityBZV/WNDduhUIw2SpZb1ZVYZw8xJ/+A0b/vjTp/Zr4W4eDlFU2eW6FYCKCS8L5sn6IhcJHuGdWfQWf4KhQjgHLOv4b+Rz2MnSXEzHnkP7emDA11/grFPMaIBpULFozRp3sM1PkrFCOAphyOUkf9IeZiwndUoc5foZjHqOKbF4gvagx1OmuWUs8Rv/Hq/BWKEYBPU15n0D7abqwcdflo4+xjV+o5om9P6vwVihFHHZONRjyILUVdl8ZOonfUb5k6f4ViHqNsAlGdk4tGfbZqGeqiZ+JQ5F9L77MPdf4KxQig1H8N4duM41rIkX9dZYvY/TDi90ydv0Ixj1FGJ9fK+Y+4IytDXW81wch/REN/df4KxQigzDnXMslr1MPYEtSX8E3/mu8jfs/U+SsUI4o6A86FHPnXBXOPXBppRAN/df4KxSiglPKvwXMvZN9fW+Sf3iVzv0d9wFTnr1DMYzTN+Zs+4gW4mIuBy9EP3k/y1+1Odf4KhaIxlEX3ox6BNo36dP5pxA+N/BUKRcMoczC16vxH3JGVoT6df/o3trePZtyvzl+hGHmo2qcctV2Zs3zjqN8xdf4KxTxGaeSvOv9KqOvacp2/vX1EKX91/grFfEYVfrkO57aAfX9tF5fp+7Wqp0KhaBrN+5ckbK1LETMfUZ/aJ1TXYTRDf3X+CsU8RpXZpFrVsxx1V/XUZRwVCkXjKKMW6tWXj7orC6Mueiaf3KW1fRQKRcNweWZvmzo4/4Xr+ysPa70GCbM3NNlr1KDOX6GYxyjX+dd4nhr7mm+o6qR7tYuzSV42RjTwV+evUMxvNKspz8o7jHoYW4KqtE+vVkWVz2jfszlz/kR0HhHdTUQbieh9c2WHQjGfoTr/4VEX7ZNF/qrzHxxE1ALwKQDnAzgFwK8T0SlzYYtCMZ+Rc/7NeucF7fwrXluv2na52mdh3Ky5ivzPArCRme9n5mkAXwFwwRzZolDMW1TxM/WUdF4YDs2HqtfWq11O+9h/68JXbngYT+zcb237wk8fxMXX3FfviVLMlfM/GsAm8f2RdFsGIrqIiNYR0bqtW7fOqnEKxXwBB5KMgC7mUhX9JHxLq6cisIxjDb/EYzv2433fuA2//YV11vYr7ngc37ntsaH792HeJnyZ+WJmXsvMa1euXDnX5igUc4IqfmsB++1a0M/9KRso3Mlddd73mW5SKvTJ3dOFfVFDOYW5cv6bARwjvq9KtykUCoEyYUkdk7xMDwuFx/ah6kI1zOX3wZ3kZdo2mfCNmRtbLGaunP+NANYQ0fFENAngQgCXz5EtCsW8RRW+Wid51QPucbfdyV113jPTl+vnmZubR9BuqN9SMHOHiN4J4LsAWgAuYeYNc2GLQjGvUVJHplapZ31dzTv0o/YpfwOyI/46I38z7Hidf0Pef06cPwAw83cAfGeuzq9QjAKqcf51FHZbuO6/stqHGcxhT9tkQbcs8nfifAbXumKbxLxN+CoUivLa8VreoRoqq316tHWremaRf42/hC/yb4r3UeevUMxjNM35G2ezkCP/qsls5oqlswMzfYdBqCvGwlP7KBSKCuAyqqFGMngB+/7qbzVcPsvXLNweO28AdUTmnL1FFLc3RfvMGeevUCh6Y7Z0/gvY9/dB+3A57ZMV2bMln3UgG0ecAX1BJnwVCkVvcBnFUKfzWcjev+LQFjNAVTj/2P5eh28O9cVozvkr7aNQzGP0Fh7WpfNfuN6/enkHLr3hLgVXbz0kv/dvkvZR569QzGdkDqfoaGYj4bgQUPXaGOXJ4byks9H5J9vrnIGrkb9CoQDQQ31Sw6IiJqpc0OUd+lL7lOwX7eTfOhDqK+a612rOoZy/QjGPUeZoanXXC9f395XwLVX7ODN7c53/8Ai+RTAvrPIOCoWiGqpMOqqF8x++i3mLvko6V+hn+PetIkIDSZO0jzp/hWIeoyypW2u6cQF7/+rlHarRbC73X4dzDtUJarKwm3L+CsU8RuniIjXONF3QK3n1o/Mv3W9/aKSqp6+2j3L+CsX4oWmXbPxKxZL3I4l+aJ+45I6bdQFMi3o5f/9547i58g7q/BWKeYxKnL9W9SxF9TV8UTrauk6/zluWJ3x959TIX6EYQ5TQPjW+Fyxc19/fJK+ypq7Tz3n64Z1zKPJnZk34KhTjiLKSzrXqzRew9688yatXQ8fp1wkuGUhU6qlQjCEqJSBrOc/C9f5VKa1ezdxlHBvR+Xts0shfoRhj+PxSHQGo8SsLmPLvo7xDeUu3mmetnH8g485gRKr2USjGD+UzfG31ySAwNMPCVvvUHPnD/ltH6B+6/7FG/grFeMKtIW/tq1VnvnC9f3/LOFaf5FXnMo7hBdy1qqdCMZao4ri0vEM5qid8e0zyYv/fOpBN8vJJPTXyVyjGD6XlHXQxl0qoem1xD6G/GRryxG8D5R0KBf21vINCMZYopyHSv7qQYymqD5LlyziaFbyQJX6HscrpuyTybyrhq85foSjBz/9/38df/uddc22GX+1TY/8LOeFb9dqYy9vmM3ztv42Wd2hwkldjzp+I/pqI7iKiW4nom0S0Qux7PxFtJKK7iejVTdmgUAyLR3fsxz/88L45O3+V8g51jAILmvbpo7xDWVu3mmetM6xDJZ1HlPa5AsBpzPxcAPcAeD8AENEpAC4EcCqA8wD8PRG1GrRDoRhZcAnFUG95hwXt/as1Y0nt+PcDxci/DmTndcL8Jqt6Nub8mfl7zNxJv14HYFX6+QIAX2HmKWZ+AMBGAGc1ZYdCYcDM+PTV9+HJ3VONn+uJXfvx+Z88OFQf6zdtx7dv3VLYvvGJXbj0xk2ZM7rq7ifwx1+7FRse3VG572vu2Ypr7tmafQ9F/v95+2NY9+BTfdltcO292/DxK+8Z6NgQ9s908aF/24BPfP9efP2mR/Cje7fiH6+5vzw34nzfsmMfLrn2AU87u6jzJ75/L2a6+WjATuRvbtptm3fgsvWbB7wiYMOjO/CNWx4BMLuR/2ypff47gK+mn49GMhgYPJJus0BEFwG4CABWr17dtH2KMcCWHfvxkf+4CwctmcD/83PN/p/63S/ejHUPPY2zT16J4w9dOlAfr/3Uj8W33Cl97abNuOTHD+Bt55wIAHh67wy+um4Tlh/QxqlHHVip7zddcgMA4OgVBzi92/jo9+7GmsOXYe1xB/dt/2985noAwLvPPbnvY0PY8OhOfPbHDwIAJloEIsJ0J8YFpx+Fw5Yv9h7jjgtv+dw63LFlJ8477QgclV4/kETfsu3HrrgHL1lzKM5cfVDSj9OfifzvemwXPvztO3HB6QU3Vgn/csPD+M5tjwHwL+YyL6WeRHQlgCM8uz7AzJelbT4AoAPgS/30zcwXA7gYANauXbuA30kVs4VubEv1msRTe6fTc5bwCAMiZk6iT8erdYa4sFDk3GUupUJmG11xjcn1svjsh0tp7dg3U+jLtHNvg7wvGefv6dftqx/0OnZelndg5nPL9hPRfwPwSwBewfld3AzgGNFsVbpNoZgVzG5ys54HV9rMzN71ZkP1Ycpg/ErwnvD8ygdIRyltLnOg7q7QNSdqHy5scz/nM3zF+Yf4TyVtd/+3xNzcAu5Nqn3OA/BeAL/CzHvFrssBXEhEi4joeABrANzQlB0KhYE7Lb9RNHiKmE30b28fxgGFHHzMPPSbUp2T0cokkVXPnzl/X8mMQr/Fc/hm+A4X+UvbnIQvj2Ztn08CWATgivSCrmPmtzHzBiK6FMAdSOigdzBzt0E7FAoAclJU1fbDO626HlxpiYn6XefVHYKeCV0ql+yripkuY7Jdz40IOdky5+vuMbNoq0X+xX5dyScw2FuX219im2MTmqvt05jzZ+aTSvZ9GMCHmzq3QuGD76EtwzBOr8l3CxP1u/YN44BC15rlF4bATDfGZLsekiH0dlP6NheifTx9FO6pL/L3dFsb7eNJ+I7cJC+FYr7BVWv0wjAPtEFdz60/ArW/D0f7hM47/EA2M8wriYPQAFd2CvcNKV/DwE34+rYVcwxZdU9hyzBJ8a4V+bs6/3qWifRBnb9ibOCW5O2FYXjcJkskh95ghkn4hteQHT5HMl2j8x+E9nEds3Gm7hG+qp7y0t2VvOqK/Mt+tybX8FXnrxgb9FuKt47EcF1Rm8v5A0WHN9SbSojz91Ah/WKmO8cJX+d7WeTvOmJXZSXPJc9Zm9TTR/sM3HM51Pkrxgb5tPxZiPwHPrI3fFJDYFh7Q051+EFwplNn5B/aXl3tY7yp25dXPiuOdSP+unIu5Qlf5fwViqHRr159PlW6tCLQ9K/rlAdx0r3ehurQ+NfJ+YfebsreekKRf3HA4KLax/ps6Lb0u9N20Dev8oSvruSlUAwNw/1WdZJ1qGea4P5DuYtBIn9zTOjIOiL/Ojn/0G9Szpvb3/N1i93BE4UbIdvIU/jyA4O+eUlWzJ/wHajbnlDnrxgblFXI9KEOtU9drp899EOB8x/Ax5qovCzhO584/0ESvu6vEIr8k+R2+FB3tm8db15A74FLF3NRKIZEPj2/WvuhIv9soKk/8g9x/oM4n+mUjw/SPlykQvpFZ45pn1B5B/eYJJoPO3T3s3vKgSP/Etqnydno6vwVY4M8YVcx4VtH5F/Ts2txz2YQK0T+/Z/MROWhIxnDX0OdtE9oMK1Sh9/AUCsFZQ+KA4Uv1xJqO6jW39L5ezK+SvsoFEPCrc3Su/3g5+r3LaMfhFRLg0SJWTK2REI5qPNvRZSeo07ax7+9XOrp0D6Z2qdI+xTmTlh0m/sWUE/CVw5CXs5fE74KxXAwj1hVKmYY2ic/Z/2hv+mzDtqn0yPhm8gfB7uGVupla5V6DqL2Cewq0D6eK/W9cZnPBannoGqfkshfJ3kpFDWg38h/uBm+6TmbqIUfiPzrsNfFMJF/lHqX2SjvUJo0db5nah+fzr+QBxDncJLuhTevQXX+PWyP1PkrFMOhXyqmHrVPPZG/7CVUmnqYgSYYtXqcXFWYyH+uyzsUSjqbYzyO3r1Un8oKgHfhl4F1/qXJ6hFcw1ehmG/I1mCt6JBroX0a4PxNl3WWdyiN/AfsM2qA8x+snr/9Patn5FnJq5DwDXxO5j/YbQd984p71fMfqNfeUOevGBuYR7Oy1HOohG9/FFPV/oBigTGDJspR+FQtVZEnfGcj8g8f00TC1ysLHfAye5V3aMr7q/NXjA3ikNcMYBhnmp2zwRm+rn1Vz+VLeAcllEPM8jKTk+a8vIMb+afetJjw7SH1tPh/z+A7KO1TovMHq9pHoRga/Uf+w0fSdbl+n+pk0IRvP2Oad9ZrRRjnP12j2qee8g7+Y9gj3wxN8oLnLWHw8g5lkT9rwlehGBb5SkyzF/k3UtsnIPWsaq+vXWl5h0Glnql36dRwHw0GquoZ6suT8C0z1U261yG1BcoHrph1kpdCMTyyiLla8zoSqI1M8srmZQ1G+/jahcU+PDCX3U61nnOt83evl7LtdjufgicU+RffEeqJ/F1oVU+Fogb0SfnXFLXX4/1dmSFQdBrDRP7BhC8PfwVzrfN3LyDX+fsi/+I23+ck8q+H9pGDq9sDQyN/hWJo9FtsbRifFaJm6kBe3sG/vRd8kWb5JK8BnVp63HSd5R0GSfj2o/ZxjnXLOMu2ZeWf+4G0ozjPQKWeCsXQMA9W1Ue0Hs5/6C6SflB0EIMWdvNFySFenzH4NRh75jryL6p9EvjUPsUZvn7H7Kt2Wgft4+1BJ3kpFMMhmxlbWRUzvOduUuo5qPPx0j4hzt9DhVSFOW52dP7VE75h2qfI+bvyTtlnWX6gH8RW5F/8rGofhWJIsPO3F+ZV5O84HmBwnb+f9glvG/QSmoj8w7RP+JhQwtfty7dqmfXGJT77Of+wDWWwIn/PYDOyCV8ieg8RMREdmn4nIvoEEW0koluJ6MymbVAogHDEHMJQOv8suVx/5B8qUFed9ilu85kZKh1dFcae6U5996BJnT+8ah//Z19+oI7FXGx6jy1760ajzp+IjgHwKgAPi83nA1iT/rsIwD80aYNCYZA75Grt4xoevkHdXoF7tvYlf4uR6xCRf5kNA16E8WlzPsPX+Z7N8O1T7SO/+BLh9dA+4nSZvc2g6cj/bwC8F/b9vwDAFzjBdQBWENGRDduhUAipZ1VuPPk7yMNnzjA4X162r/gG04pouIQvA5u378PHvne3eEOyz3f75h24+eGn8aF/24B9092e5xmW9rl98w586fqHsH+mi098/15Md+KBJnmFMr5ddjh2FAeK793xGL5/5+MAPJF/n29eV939BL51y+ai7Q7t8+1bt+B7Gx7L+m8q8m830y1ARBcA2MzMP3Mq1R0NYJP4/ki6bYtz/EVI3gywevXqpsxUjBH65bDNwzzMAtrfvHkzli+ewPOOWdHXcQVH4qNlRJuJVnXn79f5M9755Ztxy8Pbcf5zjsSzj1yeS2PTNr/0d9dm7V95yuH4+RMPLT9PlvAdbAA059u2axp/c+U9eMbidj31/MUxro7fDQx+ePdW/PDurXjwI79YUFwVOP8eA/2bP3sjAOC1Zxzt2C5tZbzjyzcDAO7+3+cl9jbk/Ydy/kR0JYAjPLs+AOBPkFA+A4GZLwZwMQCsXbu2fuJUMXbol8MehvYxp/jGLZsx0Yr6dv5lNvoWcJ9oRUPp/GPOI/SOWde35H51Kjj03CEP9/hOdZK3jD1TnXoKu4kF3OWuXgvX2Gqfojh20BLg0nb37ULaWzeGcv7MfK5vOxE9B8DxAEzUvwrAzUR0FoDNAI4RzVel2xSKhuFPlIaQOf8hWddBqJ8CH+3R+cs2k60IeytQMUDASTGjlZZj6KShaFmOpAqV0/UMUoOgnWodOzEPFPkX1T455+/SPlXZozrr+XftUaVwvpFS+zDzbcx8GDMfx8zHIaF2zmTmxwBcDuBNqernhQB2MPOWsv4UijoQmhkbQvZQDvTs+aO5qvDJEPPPRc5/ohVVrkUUSvgaJ2uuu2zZy17OX2rmh53rYAalblyUVxoMEvnHsR29+2r0w9lvta0h4etbUMb9PC8j/wHxHQCvAbARwF4Ab54DGxRjiPzZ7I/2GXaSzSByT3+5YfM5bSOdf5sq0w6hSV4tEWEDYl6Ex/5eJRvkOYaO/Fu5XaHTli/mYsOifZyd1at6+hK+4WND8FUWdT83pfaZFeefRv/mMwN4x2ycV6GQyGf4VmtvHuZBEr42RTA87ePjgi3OP6oe+Xt1/uBszV038vc5xF6VOrtOlDwMIjEjN0j7lEb+YaqoQKeV5VqscxffEgapAluk94qfR1Lnr1DMJ+Tpx6pO0nD+w6EO2geQaqUi7dNukVetUr1vO8I22+T5JHrRPnKAGZb2Mc6vE/NA5R1cGNvcyN+3hq+ETREVB9FBEr7uvfEtGzmM2qwM6vwVY4N+19XN1T4DRP6efvpB2SzcLPK3pJ45L96z74Dap5Vx/ibhWxL593D+loJlyDleWTmGmIPRdRV1lPvdvYQ4Lh887dr+xUFxkISvb6KZPEeTUOevGBv46JIyGEczbNw1SOBbNgs3JPV0t4UQquqZJ3zTbc5gI9Ef519X5B/XUt5B5kzsyL88G+S+JRTzBcMP8uz50pTOX52/YmzQ7zKOGe0zkM5/OOfnc2a5/QlcqWfVc3kjVE/kn/c1CO3jcOlDQEoz6yjvkEf+Luffg/Zx3mbqcP4Fuy3paT3BRwjq/BVjg7JI1odu5vznXucPFN9cpDOaaPvr1Xj7Dko9jc7fHmSGTfjWFvl3E86/5ZFflap9Ak66687wRTntY3H+voTvIGqfsoRvFvn3328VqPNXjA1y3Xo1Z2SYjUGknjbn3//x/lm49ugl22Scf4Vr8+cTcqdqZu+W3a9+Iv+61jQwOn+f8y8v7+By/rldLs0SMjV25hgweyZ5DfKGVyb1TP9qwlehGBJlkay3/RAJX18//aB8xqr9FxCc/4CRf8z5JK/MsZfkSHpy/k5ydBgYB2nUPhO+yL9U6ul+l5G/TbOEKMGZOO5ZB2gQtU/Zmgx1VJUtgzp/xdhg0MJuA1X1FCepO/LPpJ4ezr+S2qfHJK+ZLPI3+/qP/O11aYfz/saebsyIY/Qf+RfUPuavW9sn/FvNdO18QOzJD5SuJhbKVZSofTLaJ9jrcFDnrxgbSJVHFXRrivxr5/zjYr8TqUa/CvUQquppdP7GsecL3hf76F3eIf88bORvaKhOHKPLnL3lSPQT+Qc5/xLaJ8lxyLeEcCLZe3zgTam4cpj83Czpr85fMTbot8hkfWqf/o/38fJlap+c9undd2iSV8uhfbLI39NHP5H/sJy/KTTXjcsSvmWcvw3TtGv783SSV4D26caFiWtuNF9mQ+h+FSP/Iumvkb9CMSR8M2PLkJd3GPK8dUgAIWYoZ4nYfN9Eu5+Er7+NUftMd91JXsX2vZZmrJPzn8ki/8Q5+yL/8vIO7vf8unzVUn2Y7saFtm77snsvnX9VGbDZo5y/QjEkcg67WvtB1BsGNpdcE+3jTL6S9tKwVxoAACAASURBVE0OmfBl5kxVMtOxB5dBaJ+4Rs6/0x028nc5/wDtg/D9czn/ZE6A02+JDdPifklbC7OMHUURoGofhWJ4lESy/uZhzrsfDFLewGej++bi5fyrOH/vBLI8wsw4/5IcSV/lHYamfdLIP9X5m9yEdb4Sc9zLzWifgtQzPP1vphsXBopC5F9ig+T8y+6NnStJacdwt0NBnb9ibFDGYftgnORArsvzEPeDkIOWXctBpS+df9pmQjhRyXfnnH/4+nstzWjsb0c0mNpJHGTs6abRdtun9im77oLaJx08Halnovbx9zPdiZ0ZvsXIv5T2EZPi5O+mVT0VillAHslX5Pz7bB8+b//H+OWYZZF/H7RP6nwMx29sNN0Zx+7mGCSqVvVst2igwU/23xFSz27Mlt0GZYlT9+wZbRZ7FnMJmDrTjb2OWaLs3svrKYv8fVU9R2olL4ViPqJfzt88zIM476E5f88xrlrJ4vwHSPi2rcgf4ch/AM7f2DERRQPdP9n/TDqSJAlfeGmf0pmyBdonHzyLVE4Z52877b4if0n7xP7PQKAmkkb+CsVwyOiSPtU+w3LWtSV8Hc7fUvv0w/lntI8d+bvOfyjOXwwwg7w5SWeZR/5xGvn3TvjaUbo/ui4WdgtTfC7n7ysFUTXyLyt94aPTNOGrUAyJfmf4lnHeVc+V9NP/8d6Er8P5S/Sl8xd8fN53rmZxpZ6+8/Uq72Dsb7eiga7fon1M5N8djPYpJHyzxVzg6PxLOP9u7ET+xbZVdf7yDcEdQ33zIzThq1AMiTySrda+jPbo77yDRP6+fpK/Pgc1WMLX5fxN5O9y/sU+elb1jA3tMxjnPy36t8o7sF/tUzpT1jl9ljvph/PvuJx/URlUdu+nA5G/O2B0PLSPJnwViiHRd1XP2KZZ+oHN+fd9eEDtEx6M+qntIykZAwZnEbFx7L7ko0FVnX8rXV6yX9gJ31znH6rqWVojJ1DV01feoay2j/s20R/tIxx+ScK3K17dMspfnb9CMRzKIlkf8hLKw513oMVcfLSP+evpztTz72cxF5v2CXP+Sb92H/0kfAdT+wjO3+j8Y0Y3RqC2j/2dXT5HIOP82eH8y6p6ujp/9pR3qCj1LEv42pG/oX2U81cohkK/K3kNFfmXOM5+zi2Rq49KaJ8qVT19tI+wc7pbJfIvP498uxhW6mk+d1LevUpVT1fFY7U19Fkfhd1czt/3llA+ySug8y/JG2jkr1DUhIwzrzjjtt9JYda5EHac1c4dPsbn3wfS+bfsyJ/7iPyne9E+JuEbDZbw9ZVDyOr5e2f4hk9SvP8y8re3hqWe7gzf4ltC5fIOkvapEvmr2kehGA7cZ+QfDxH5W/3UJPUse3OZHCDha0/ykjN8i7mFoiSxV8I3+TsxqNRT0CTGISb1/Bktj9qnvEyC2zb968zwLeX8O7G1L46LQUQdOn97HYTk70iqfYjo94joLiLaQER/Jba/n4g2EtHdRPTqJm1QKAz6VfvkM3wHP1c/57PO7Z3hG+4vX3x9QM5f9OvW8wfsiBSorvYZXOopzt3N30i6XFXnX3SiBhbn7wxwQbWPowv1Naus8y9N+M4e7dNupluAiF4G4AIAz2PmKSI6LN1+CoALAZwK4CgAVxLRyczcbcoWhQIozpDthay2z3CB/9A5g2xbyT7j/Kucy7RpFXT+yXYjs5Rduc61F+efv13Ux/mbWbZezr/kHCF6pmzNBBfTKe1DlCfH+5vhW62qp2V3Fvk34/0bc/4A3g7gI8w8BQDM/ES6/QIAX0m3P0BEGwGcBeCnDdqiUOQrUwW8/4ZHd+A/bnsM73nVySCiUprlR/duxRevewjveNlJeOTpfYgIOO+0I/GTjduwe6pjn1cc/uTuKXz0intw6lHLcfCSSZz/nCO9tvgcySXXPoBHnt6LbbunCvvaWeSfOO8/+trP8NSeabzg+IPxzpevsc7/8SvvTY4R3Pm31j+Kw5cvApA4qi079uFPL7s9t0c4LCLjDLnAR199z1Y8sHU3DlmW9DXRiqyI+EP/tgFbtu/HX7zuOVgy2cKf//sdeM8rT8YXr3sYt23ejlc8+3Dsmergsz9+MDvGvHXMdGMQIVDYDXh853588gcb8We/fIp1zz/74wfx+uevwqlHHQgg/z3u2LITP7p3W9bur797N17xrMMKfQPJ20fMjBYROsz47obHsGXHfquNuUf/um4TGMAtD2/H6ccciEXtljVv4aEn9+DSdZvwp794SvkKZDCcf7DJUGjS+Z8M4BeI6MMA9gP4Q2a+EcDRAK4T7R5Jt1kgoosAXAQAq1evbtBMxbgg43oDz9sVdzyOT161Eb//ijWYbFMh8l+/aTsOXjKJ1YcswQcv34D7tu7B6ccchO/f+TiiiHDeaUfiH390P7btnnbOm5/wL75zF75+8yMAgHNOXhl0/h1PSPjP1z1U2HbARAuvf/4qLF2UPMrdmPHI03tx2fpHQQTc+/huy/nftnkHAOC5qw4slA14fOdU1sePNz6JWx/Zke2Tzv8VzzoMV975hLe88jdvfgQ3Pvg03nveMwGY8g55H8ap/8YLj8WKJRP48vUP4+w1K/E3V94DAPjBXU+AiKzzmXvRiRkR+Wv7dGPGT+7bhn++7iH8txcfh8OXL7b2v/HTP8WGPz8PQPJ7HLpsEtt2T+M/bt9itfv+XU/Ah06ccP6tiNCJGZetfxQA8IzFbeza38n6BYAv/PQh3LFlJ7ox46s3AmeuPgivPOXwrK+3f+lmMAO/duaqbGD89bNW47L1m7F3OidAzH0bdjGhEIbi/InoSiK63fPvAiQDy8EAXgjgjwBcSn2krZn5YmZey8xrV65cOYyZCkWCjMP3e3/zIMpyAkD+UL/2Uz/G2X99FQBbBjrTjS0H1XFmjkrnH+J+XUz14NQNTj9mBf7Xa0/LqBBJ3yybbBciS7Pvzy84Lagi6abXZG1Lr/dPXvMsnLH6oKydC0PN5HmFXOfvljjwSWljzs/15hcfl/Up94fKO2S1mOKiBt9K1jLwujNX4bBnLKqUIzH9x8zWZLo1hy3Dq089Iu83ztvm15Zeq5NYBpKaPabd2845ARecbsfA+X2Zh7QPM58b2kdEbwfwDU5+hRuIKAZwKIDNAI4RTVel2xSKRmEepdDzbh7QmQ4Dk8CMifxL+oxjxnSX0c7eKriQ+AvpustrwVRzSsZ/Zwlfzp3gRDsq2GL2tYiCEWUcF9U8xu5IHOfjzKe7seX8Jlp5PX+3xIHpM3QfjKPtOCcK1fM311pYpAV2fiNmBlEexVeBGawm2hEwlfdJThvTv2VbXPw/YY6X99UdizPOfz5G/j3wLQAvAwAiOhnAJIBtAC4HcCERLSKi4wGsAXBDg3YoFAB6F2ozvsk4KRPNl+UrTZScLThinJo4xl0EJD9fuOPpipG/cQyGwnEdb2gSURSFq0V2Yy6cP1/MnmDmhvkj/9hSzbREwted5dpLSmvmLnScgbAVoH3kYOJ2KccLZjOIUeXIP/bMMXDvXzb4uAOuGJCt64hEGYyomNZturBbk5z/JQAuIaLbAUwD+K30LWADEV0K4A4AHQDvUKWPYjZgHEKQ9nHoCel0QjRCHHPq8FK1TVyU/MmvvqqNPvTS0RsYlyHVPnIGr+SQ5TlbUTjyTwY02zYTIUcEMIWVRTMm8hc2mGZufZts8lbgLcc4f/deTAR0/hnVEqMwwruRf9Rv5B8nx8lZ0a2IrNNkg09hwPUPlET54NyKipF/Vwy4TaAx58/M0wB+I7DvwwA+3NS5FQof4sz5+/ebhy2TFsaSn/e37TInE4BagnZhe/ZnSNddGvlXdf6G9ski/7zfyZaP9uGsfcipmAHNd1xEBKa8nYuZDltvH1Lq6da0N45vf8cf+5l6Ra6DDhV2C9XscY9JnD8ltEvF6d5mcJkUzj9y7MgHn2q0j9xu3kTccyb7KpnYN5qM/BWKeQV3MRQXrvMvj/zNKz4szt84Ptk8FPmX0fpVaR8DEwxLpzrRioIJ36hn5B9y/si8kW/wSmrgCJ1/K0/4uiUOjN/dP+O/1skA7RMq75ANyF7aRzr/JJqOKPzW4e2f7XpILfLPgi5E/k7C19peQvsY20ZR6qlQzCvktI9/fz7JqRiputGnq/ZhFrSLo/aRA4f0NVVrwZTBRO8y4Wv6nWhTsQSBiPxLOf+A8yeR8A1x/qGErzvRKYv8ZwKRv6F9nIvwl3ewfxPXMqmGApDRPpXVPul9NW8jsk95TUAxER6K/M2AYuxx38TM/RnFSV4KxbyCuwC6i0LkLx7YKSc6Nbu6DudvnFqI6qk94Zv+bXkTvsXI34o0g2ofThRPnuOsCNpjoinBIKWeQHLvZZ9SnTPVw/m7P1co8reWZ3QOirI8Rf49Iqo8yJqE76IJQfsQ2fSekHpaton7YfeZ/39wKSTZX1MZX3X+irFBryCvmPDNHcNU15849Tr/OFzbp/aEr1H7iISvRfsUks+5swlFlF7aR0TMVBr5J049TzrnTteu1Jnfi9CcBp+TBwKcv1DUuG9e8pjYifyrJ3zZm/B1bZB/5bFB2sckfD1Szzzybwbq/BVjA/P89Yr8jZOS6hQ38pf8cjKxKe/b1aVbSd6qCd8hIn9z+klR5jnKeHrRvkTn716DSYxKPbqPypjuxJYjbrdypxtM+AYi/8m2X4ne9izmIgcct2BbYjcyOwAjWe2P9nETvq2ILFVRvwlf+ZbW8gzGUl7bBLSks2Js0GtNXuObjNOXDtCNTjPKIlWqGN16N+ZCEtF6C4j9A4GL6U5cKdFnHEMkkrBdJ+ruegafKApHlF3mLO+RbYvN+coriJq692awMLSP6/xlbiKU8PWt2JX0GYr8Be3jxP5Rxvmn31Paxx3kQjCDVanOv8+Er6S+5OS57DiZZG8A6vwVY4dQ5O9ORrIif0eOmEetovpkWv/FpUxCkX+vhO+iQOQrkUX+UucvOH/AoZoqSD0NlSXREZG/TC67mHHemoyzZC5J+Iaknn04f+Y8RxMzF3X+ztyE/hO+yb0r1fmLwcc6Ns4HTzjty3T+5np0GUeFYkj0nuFrc/7SWbk0TDeLWnPHNdNNI/8Az558Fn30iPwXtVvB/QZlOv+Jdh51u3b7pIUS7mBnAmQ5M9g3eMna+0BO0SSRv53v8N1DiX44/+TcZpZ18TfOB8f0OoiSCp0VnX8W+bfthK9EkPbhKjr/Ir2TU1SVTOwb6vwVY4NeUk/jjPPyDpy9chdonyxqFZF/x5Y55m3FZysKD9s6UzHyN7F/pvMXFMOkL/KXCd8Sp+JSMVUj/2kxcBLJiLsY+ceetyeJyUDkH1IqyRIbIZ2/dKhRBHT70PnHMRzO3yndMWDC1zh+95IyeW0lC/uHOn/F2CDn/AO0T/qwdQTnb5KOrvPPlCpW5B97KSX2RN7uZxfT3diSFYZQjPwl7ZOXnHDP2fI4Gwk3GpeJUllHyIWkfaSCpcD5x3lSOBT5+xK7gH82rDmnOVdohi+bN5h0EPNF/r4XC5NTkJRTLTr/WPTjnDf7v6SRv0IxHMzjF/K5kvbhlKYwHK+kfZKyyclnGfmbipYurMg/wP+7mO7EWFyF9kn/trwJ3+K6vjbHHPYq+53BzgyIhisHik4uieZT+7txOos4d7ohnb97LoNW5HfEIcpquizy90g9Q4XdfCWjDe3TbuUDWqLzF23SL6GEr/smY+owmXsUopFCk/GGhTp/xdig1wLuUudvHrxFWeTfFe3yY6Yczt8X+Yfq+ZSv+cp9Rf6URtmSSy9L+EY9In934lXuNMNVPa2lFzuxVTY6Zi7q/Ln49iQhKSZ3uzfyTwcR7wxfR+oZRSbyLw48oXkEccxZrsDXLpzwLVYETbYjXZA+2e6eNU/4NgN1/oqxQabzD3Dtuc4/T9oaByp1/vLh3u/QPr5IkjkfeHxRuA+VE77CNbTSSLY4wUoOPmnbKKzzB4pUjKzqGaJ9pHOf6cZJ5VARcbsL2fRK+LYiv5MP2W5z/n7ax5hMqRP3jb+hBeK7zNY1tYisNwxJuUmYCH/CyeFkfabX6F6q6vwViprQayFxk/uT9fl9nL/sR26f7vidPyAHnj44/ypST+EXonRxkHyCVTHyt2bqlnj/UI6DRDTu3k9Zr3+my5aCpSzhG5rhG4r8Q3SQ4fzLEr6yto+vpALgXy/ADFYy8nePl8l2iTzy99A+cT4Bz/09shm+yvkrFMMhc8A9Er4znTjjuHPOP49OyyL/kD/3KUFKdf6d/p1/i8hKLoZon0xd0kfkL/nnViDyl1LOaRP5p+cwORTZX5XIv+UxMqGsitvNm4fv9w1JPX0IRf6GomnJyN9pE8r5dOOiekm+TQAlkb/XyuGhzl8xNjDPZVDqKRK+ppKkeWBldCoduDXJq+t/+OW5Q0s6uhiI9onI4tInfbSPcDZlcOWXcrZpRuUUnL+H9hGR/3QnRPsYGaltg6RYCtvJ/i7P79ZWknAnefng2x7H+b0zu6VtZsWyUGDRiePCwvNx+pYWZbSPP4egkb9CMSR61vPPdP55iYYJH+0jI/+Ow/n3mD1sR+FhW2cqSj1lWBhVTPhmzqakW3fWrRX5B3T+LucvSxYwEs5/okVoR3ZuwpzLHeyCtI8zO7ntOn+P1FOWewZSyWrI+Xu8rVl/QNokdf7mmkKDf3Ltxcg/eZtIvhd0/iI/0QTU+SvGBtkkr+D+PPI3zn9Rq6j2kQ+49H/T3ThI5Zh2UlceHCjiJOHcT3kHIOX8rYSvf4ZviGbw2evaSiUJX1vtw5ac1HD+E62okJsw53IHu1DC150QlTn/VErKXOT83Ql+vlo62XkDnL9Z/tHY1HIGIFlUz8VMp8j590r4mvpISvsoFEMik3r2QfuYxTsMZdGKiouiG8x0ekf+VZZxNBF0tfIOjtpHRv5tE/nn7S1n04db8UX+xYSvzflLSWayNGTiALPchHO8O6+hJWSl1nZnhq9JbJvfLCns5tjv3P+I/BE+UK7zl1RUJGr7JCuWhQf0mTjOaLisz14J36yYnkb+CsVQyDn/EO2T/JWR/4TD+bei4upYBtNpRUv/uYu0D+BP+ubOv//IPxYTrXycv+VsPD4l5Gf8CV+/3YDk/JPvzMn+iVaU8uPFa1/sRP5RBG9U3IpQTvt4pJ457YPsOoK0j7OdKHljY7av3xv5h5y/h/Yxg0Uw4Wvetrw9Dg91/oqxATt/XRhnNN3JNemTjs7fRNc+hGrUADLh6zglT1/mLaOfSV7Gtl5VPcucDQBMeKJe2UeiFLK3GXQKnL9dU6eTcv4RFevdRFSs4imjbGmXS9mYRKp58/BN8nLfvGTdIReu2mciirJgwFL7RJT9Z2q37HvvYqbDhYSvyRHkb2Io7De2NgF1/oqxgY96kZC0TzbJK42+TVQbUVii6VbClPBN8pLnlDADTz/lHYCi2sfP+ctyAUWv4joo185Snb+Qcna6bBWPM1U988jfdpRmu0RLnEva5ZamMDRNmdrH2M+cX0fVyL/doqzvZEBKtsvj21EU1Pkb29x+szyCoJF8Nmt5B4UigGvu2YqbHnqqd0Mn6edClncwUayb8A2txwpUi/zdhV58A1F/kX/uGKKomtonU5d4fIpP4w7Yk8NCi7lYJbC7pryDkHqmkb+poy8d5WQrKji5SOj8pV2hyN/W+feifeDNJ/juQTsiMfjbtI95x2i3ytU+Jgfi2iQjfxdV1xoYFLqMo2Lk8aZLbgAAPPiRXyxt10/kbx52t7xDHPupGiA8Wcmck5kLC4b7HvBd+zsAgCWTvR9PK/I35R3SPifbxQjdTvgWEVpAxZQ+Jsuh27a7nP+yRW1rZu1MJ1X7pPSUvBUTbX/kn9E+hbVzyfleXtLZXcWtbJKXa8dEKxKRP7yRep7H8HbpjfxNVc9QDmZkZ/gS0elEdB0RrSeidUR0VrqdiOgTRLSRiG4lojObskEx2nhg2x7c+/iu0jZP7p6q3F/mEEJqnyzyz3X+bnmHLod53VCBMiDlvH2zPz3OYsOjOwAAzz5yebC/DMIxZBJKh/aRTrZXwjdI+3gi/yLt45Z3kIXdkv2TqZN3k6MTrSINY0X+Du1jRf6Z88+T6gW1j6PzLyvv4Kp9EtqnmPCWtX0yGwLe35S4ljC0T67zd/ZnM3yb8f5NRv5/BeBDzPwfRPSa9PtLAZwPYE367wUA/iH9q1BkeHL3FF72f36IJZMt3PHn5wXb3bZ5h/WdmYPSOF/kz8z4+x/eh6WTrby8QzfOqj0a52+omNDCHEC4Rk1ynuLyjoD/LWL9ph1YvriN4w9dmm0LLTnoFnaT5R189IyV8PVx/j0SvkRhtU+hqmeA809KKdt2JRJQuz+Z8G07CV9b/RNl5zTnKkb+uR3udbgocP5RhL1TM9k+OcmrYINvvcbUtiKnDzvh60b+DSd8m3T+DMCELgcCeDT9fAGAL3CSebmOiFYQ0ZHMvKVBW8YSzIw//Ndb8frnr8KLTjxkrs3pC9+4eTMAYO+0P5r+1FUbMdON8fEr7wWQcMZr//eV2LZ7Cq8742j87stOxDdv2Yw/fNUzs8FABv4798/go9+9GwceMIFP/GCj1XdS2M2O/G94MM8pSOVHqM6Pi3d95RY864hiJP+mS67HqUceiCMOXIyXPnMlzlh9EG59ZDued8wKyzm1iND1vLLYTjDn0mVdnMIkr4CzAcJLJ9rlHZJthfIOrs7fmuTFFucvcxNA8vv5E75Fu9zJXxMO53/Z+kdx8NJJq68Htu3BX3znTvzyc49Kr8M/exgovv1MtHLOX547ijyRf8cfGJgciEQh4euY80/XPpDZ2gSadP7vBvBdIvo/SOiln0+3Hw1gk2j3SLrNcv5EdBGAiwBg9erVDZq5cNGJGV+/+REcvWLxyDn/3VMJ733iyqXe/Zeu24SHntybfZ/uxtiWUkB3bNmJK+54Ap+66j689ZwTsXzxBIBc6REz43e/eDOu3bjN27dV3kGEd695zhH4zm2PZdHdRJrkm2xHiGMuTfhed/9TuO7+YlL69s07cfvmnQCSweOM1Qdh01N7sfbYgxxKB4BnbJFuQXLpUiljJXxZTipKcNrRyzMbQqtnDVLeoUWymmaeB/DRPu1WcTZvFOURtbTLXYXM5fwffmovPveTBwEAqw46AI88vQ8AcPE19+OklcvSY0qqehbUPjnnb5V3ICoc4+Z0rPsRET7yuufg+geewjdv2eyRevrtmZecPxFdSUS3e/5dAODtAP6AmY8B8AcAPtNP38x8MTOvZea1K1euHMbMsYX5DztdcZ3S+QSZvPPuFxTLyYcvyz4fsXyxlaCT6ho5xf+BbXuy7SuWTFgKj5mOoH1EFHjUgQekttkDw2QrwkQrKtTD8eHXzlxViEqz82aLnzMWTbTs4mUBD+CL/BMHnzu3bjDyT/6+7oxVeOvZJwAoUft4J3mFOf+YbW7e1PNPVD1F1VRI6mnuv6X2cSZ5mTkA0hxD0/3PX3w23vD8Vdn2q+5+AgAw2WpV1vm3o7zuv1XPP8rVPu4A5CLm5Pe48KzV+NgbnwcgV/uU5WCA5iZ5DRX5M/O5oX1E9AUA70q//iuAf0o/bwZwjGi6Kt2mqBnmFTT0H3I+Q64F64Mc0KQqZslky6rH7y4gAhSlnhOp8+7E3ewYl/YBgMUTLatPMwHMTFwqo30MJtv+ejWyX1MAzZZxBpw/7DZdzh18RvtIzj8u9sWi/1DCtyP4Z3dJRNf+zB5H6mnq2ySzpN2Er8f5R5QNsEWdv93ORV5B1KZ3fnj31vR8doL5n960Fp//6YP40b3bvDr/7FxEWW4i8kT+Zc+aOY7SRLihfUJ5FoN5Gfn3wKMAzkk/vxzAvennywG8KVX9vBDADuX7m8G0xwGOCozzLXuNNlgymU+GWrKohWkh1bTW3s3+Fvlm+YBLnb+kfUz5AVcGagaPMtrHIHFy/n0zaWG4TmwSo/m+ED9tz/BNa9Ck0WSQ9iH7WBbyz1bAEVnLOAYjf/t7MeEbY6JtEr7F38AdFIly5y/tkvMHAP+AZX53OVgBwL50gJ5oR1bkv2LJRPb/qKD2kclmcV8jygMJ01fZsybtkPcgn3Ed8vKjx/n/DoC/JaI2gP1I+XsA3wHwGgAbAewF8OYGbRhr+KLfUUFO2/Tp/Cfb2LFvxvvWkys+7O0TLbIW2pjpMmbiksi/Yxd9Sxw1V4r8TWEzH6ZlQblWVFDy+BCifexa+g7t41H75GUUyiN/i05ynP+0o3aSNrBI+Pp1/naEbj5nA2xk2yqv20dVTXXzipi+ezfpDMJE+TmLOn9pl6jqKdplE80CCV84duSyXBRyMC5GTu3DzNcCeL5nOwN4R1PnVeTIOP+S/5DzFb1oH+m8DxC0z9LJFmZEbZ4Zi/Pn7O+05fwjK8KfDkT+5rPL+ScUArC/ROop+whRODPdfLWryVZUWKIR8Ek+nWiS8yRiHvnnrZPI3+aYmSEif79tcuH3Kjp/09aifVLO36/ztyN/Y4+ZqOZSL3bkX3xbySN/v6qn+DvkfRY5f1tmmkX+TiTvuwcS1uAmZLk5jeQ/Tss7KPrGaEf+YdrHXRJwyYQd+Yc4fxaRv4xSJ1pRFsWbY8xxsrKmedgztU9k0z5lk7wMJlthiWGyfGSuJLIpHb9zdiN/w6Un9ESyvZDwdSJNRj7RKORo5ALuIZ2/O4nNSvjGiYIqr+fv0fkLb2T8raTW8n3+ev4+UOCaJlq2zr8s8nfzDb5lHN3Koj64M4IznX8P2qehwF/LOyxkTI9wwtc4fR/t4zqZAyzax+H8pfMXOnkr8m9HmOgI2qcTZ+UcFouBxTiLjPMXtE+Lh6d9rLIS7cibUJyICNPiGNmTWWvARP6ylr6BT+fPgnoI5R6z2aaUFzYrlHfoFCN/uZhLVtLZ5Cac2j7y9zE2Zglfx3FatE8oiQLj1Ivb3QSzajV4TAAAGKlJREFUpIeKOn9/vsGK/CslfOWbglwgxv49fNfQBDTyX8AwcsUQdTKfkc/WLJcUtiOyePmli9roCPpkxlp+Me9D+q1JD+c/3Y1BZC+6bT4aJ5dFpe0+Er7tMO0zLVRGCeefwzjcssg/SqmEbmxHqEWdvzk25eORO6bekX81qWdia05lMOfLOGa0jxX521SOW9OnoPOXtE9Z5E9+pZRLM8kqn8UZvrZdVuTvTPIqk1W7OY1M6pnp/APX0FDsr85/AWOUaR8Z3bv2y1mUCeWSPxwHGKlnx8P5B4r6+Dj/6U7CT8NxrtKeXOdPmGhRJZ1/r8jf2O2qX9xI2KCwgDsLnb9x0iydP4q0j9wWsK3fqp7GnlwWCru8Qw+dv/lsdP7yN44imx8vp338ZRwmHPrNivyd159Q5N9yBgXADjZcuG9ypg5T1k+FpH6dUOe/gDHKtI9dJMy2307WUvawtiLConaETsxZCWaf2seFHEDMg7ZnuoPJAPVS1Pkbzr/3fZ70FDDLr5OtgSWU8JVwI39ZLyZLzDq0T3ZNJipHvi3kR01VT0PlEPnr+bv2mP5m4hjduETn70gvi7SPO8NXRP6BuQnmGr0JX+cNTL4h9Ob80+uLCEZAXIXzt3IaIuEbKu8gbWsC6vwXMEY58pf8sUtbyeuZbNuRv3EWpiaQlTAOOv98ADELqOyZ6mJRuxUoI8DZceack+1qM3x76fzzOQQ2tRGO/KV9OZdu6fzZpn1cqSdz7mBCtI9bXtiUj5aY7sbWAjRy0ZVpIY+VuQmDSUd9k9E+7Qqcf8kkKQpc06TzBkbInXqp2icKlHfoU+dv1WHKxuJQ5K+0j6JPLITyDu5n97tL2Uw6zt9S+5TQPub5NhO5dk91sMiJ/CPXkTmRf2iRGIl2mc6/Izh/Z/3ekBLFGiAM7dNnwhfIB5Fg5C84fyDXqUvMdGJrHV5pw5RDZyWDVH6sq77JBruMEsmdPjmcf0g9lVxjWOpZ0PkHFFUTAZlpKxKTvKpw/s7/peS3Ktbzd4vrNRT4q/NfyMgi/wr68/kGGe33cv4mGUjIX9H3THcKbctoH/PwGnXPnqmE9nGllLJP46An2xSshlk8VxntExcoJYOQBt1tE4skomnadWkfTx+UDQiByN9x/kan7tov1VGRkHoaGay51+66CG0n8ncneZm5CHliOj9v2b0n+AcHtzKosRee9m0n35DP8A1Tgj64OQ3zW7mL67i/vdI+ir5hopBRpH3chUEkpq2EL1l8fUb7TKWRv2jLgdBcKj8s5+/y7umXbH1f4aDK5IYSvjIGBjNdzmcPu86/otonT/hK2idvE3tW8pL3JTS2yEVQjB3uf6uZLheksW7kX1bPX5470/mnA2wntpPY9qBcLvUMOU9b7ROO/N1JXjLha64gc/4lgVbxLc1ZXyHdvWjCXrtZ1T6KvpErXkbT+ZuHoSrtQ8glm1nkH/eO/CfblEXUZlLXrqkOFk1EBTUN4Kd93GgthMo6fyeazTXoPdQ+Wb0YUYDNpX0cZ1OJ8xc6/6SdfxlHd1Kc6S53/mTlJgzcyW9ujiOjsoxMVVx3eeRfbdEWEgXgfFU95TE+nX+1wm6S9kEh4WuuSSN/xdDoVSJhPmOmy1ialm3oSfuIh9BMvNpnOH9PYTcDSS14aR8RjRLlag2X9nHlpmVIVCb+fbKaaDHyN7aWR/6GS5cUiSv1dJ0NQ3L+/uuQM3yNPT6pp0X7EETkn/we2TKO7Or8nfyKI/XsxLG1rKP0z+Wcf3h/WO3jFHZr2ZG/+Zro/NmyoVznb9uc1WFy3mYm2+r8FUMiT/iOZuRvZu4WaB9H6pk5SkH7GGflq+1jYArC2bRP+uYw1U05/zwadHX+JkKblDb0gKsvt6/ZlnpKBMs7yDZRvsZwqKpnV9SSkZE/XAfknD/2cP6FhK8T+cso2chgswXcnci/oPN3In/zxhJFjvEYrLyDPIfpriV+awl3FTFf8n2gqp7sr+pZTPgq7aPoE6PO+Rvn7JZ4kA7dpn2KTtMq7+AEZebNQjqeRalUcbeT8G1HkXi1L0o9qzr/cs4/Fs7fbpNX3QxHhYaHN9GkrKhpYC0ekm5jcCHyd8/fcZx/FHkSvh12In+R8BVUWSuitLRBfuxE2z/JSw7mcu6CHfmX3HsR+bu33VL7oGyGb6C8A3k4/4q0TytK3hpiFnmMdN+itsP5a+Sv6BejrPaZ7sTZIi3um4scDCbbdi1+N2L1LeZisHRRKz0mf/ClVDGReiaf20KKWKjn344Kr+oh+BYtkba6+QSDcGE3O5pk9tXzz9tLmgHZ4CA5f2TXJJEtfC4ojwLtEztSz8ij80/vo0v7TLrlHcwg1M4jf0n7yLvQi/PPflvHqYYSvu6ksZDaxzfDtzNAeYfsxTVE+wR7HA7q/BcwjJOcicP/IecrZrqcRf5lk7xcvt11mr6qngZLF4nI31H7AMBkuwXz6LUF5eAqcvri/EsTvnmpadcBZMnIEg24LO/QklLPnlU9c2ohj/zt8xeknhEVEugu5y+remacfyrpjD1qH5cTT9rnTjVJ+Np2yrY+SKcuCwC6xxHyN4FSnX/k1/lXqurpDG5G7ePq/Iucv9I+ij4haZ+QzHG+ohOHaZ9pi/bJZ+cSFZ2jHDjcyD/j/AXlIKNDmfBtt6LCq715SPvh/CfbYZ0/kCeqQwnfAr8tvrqrQ2XLBRYKuzk0COdlGYyjcd+guoWEr6e8Q4dttY9P6pmWcfDV83cXO5H3wVyTb4JaaVVP5E59cdt/T01/oXvs0j4+nX/f9fwjWc/ffkNb5NqptI+iX5j/iMzhhdDnI0y9/jzydwu7hWf4Fjj/EspLcv7mOZZ1/cMJX5vzb/fB+fsif+ls9kwZ52+3MV97ST2N2sda/CW0gLtX7WPstM/vm+TlU/tMCqqsJXT5ks7KVrFy1T5WAjXfDhi1j+Tv87blVT3zfYvdyN/6HUhQa/Y9tovK2QlfcwV55B9+zuzrE+UdMsrIP8FPE76KvjFTUh9nPsPYmnP+JbV9BOUidf6+tm6kamiAScHnS4dQ4PwzOd+QOv8CrZAfuzedn1Dg/ANRaSHhy+7qUNSXzt9nkzlOHuMr75DV64/ye5nr/LuZ/V6df5sKCVFpR7ZGQS7sylAWGRPlEbXL+Vu0D4XvsVtOWiZ8Mxuc/xs+2G82yTWxSPj6lg5NjAtf3zBQ57+AIR3fKMk9jd1htU+gvAPZkbvbNqT2kZSOjM4WtfNJXu0oj9hDJZ2rwLeMozx2T8D5Z+UdClGh3cZw6TI6tev5F9UlidonfwtoRUUaKy/pnDt2X3mHZFZ0fi/NtWa1fQzt49P5O1LIpH0aFTsJX7cWfwgkbJfJ6EIf8toKtI89KMncgKFT+63q2Yoom4BYiPxnSeevK3ktYMiIeZTknq7zL5Z0Fk7DqatTTPiWcP6LhM5fRKsGttQzjzp9JZ2jqNqbVaJ2cbeJyH+qa2nJDapF/sjLOwT0+D51CQvex9TQcdU+RsUiHWSR9mHrWmTS2dL5RznlYSCltPb12pE/HNuB8sCYKM95LJ4oi/z9+n1js4EcgHz29pPwNW1z559G/iUDfJ1Q57+AUVYZcz5jOnP+vWmfdmTr/N0Hx17G0UbO+YsHOiAbbUV5YtnUCzLJ5YlWhKhiTsUf+efn2TPd8b5FmFIJbhRocf5OwtccZ9E+VsK3yPkzkho6E46Nbm2fiMgaTM152w4/7s7wNfc6SXbm/feifToxJwNfnJ8/O09paJwPUgcUnL9slb/1uf0VFo8vURzJWlIu3MHCtM3yIsHIvxn3r85/AcNy/iX/KecbTARkdPgF2kfkMibbeXkHoiJdYZV3cG7Bkkznb/PUBosmosxxSCmiqTuUOah2hCiuNrj6Er6Sqto73fUmj025Btcxya/GKcmJXG7CV6pLDBLO34T+aeTv2NDJOH9/5O+bmdyKirV9zL3ulfB1aZ9uzMlg7Iv8e3D+Mfsj/5DO380h2PX8/XMu+q7qaUX+ybZugPNvSu2jzn8BY9Q5/wMq1faxHVVR6imdvzPJS6h9fHzvZCvKnF5bTEKa7saW5G+yRehQ0WH74CvvYEX+aU0hF6a0gesH5PeWYx9g5J95G18VSTnDl5EMIoVJXjE7VTfJqhbqK0UdrOoZpbmJgtTTvl7AoX1E5O9ObguBEHb+xcJuxe2Jbf7IX67h64oBfHATxKZtVi02FPk3RPyo81/AkKWPR4n2yTj/iVBtH395B/Pd7stOdkoYGiDR+Sfb5AM62W5lD2Rb8PAzXbvIWCIV7f1mZVbncvXl0mGGIn+zcHrB0XkUJzPdWPDXOefNbKtLMqfCkv9PnOykM4h25PKPSKSYsRX52/JXY0+B9ilJ+LY8kb8ZzN11CKR15ZF/PviVJnxJlK4o0D7220xOqeVtqtT2CUf+6bHp/Vjkcv4NRf5DqX2I6A1EtIGIYiJa6+x7PxFtJKK7iejVYvt56baNRPS+Yc6vKMfIcv4dI/UM6PwLah/jzHpx/raDNnSLXGDFivzbOe3TjvK3gxlntmlVqadx6u6KVQXOv1182luRTaMYWJG/0Jr7Er7mWoqRv/3ZR/vErvMP0T6ysBvllEWe8CVvwtddxtGc3kT+nThO+sv4dnEPKkb+Rc6fvN/dnIxMsrsJX/N/yq375IN7/0zbPF/gj/ybwrBnuR3A6wBcIzcS0SkALgRwKoDzAPw9EbWIqAXgUwDOB3AKgF9P2yoawKg6f2ProokofUjCzn+yFeXFzqg4OcnS+cd2FJVLNSMv37uoHWXRmNT5z3Ria7Zp1cJumfN3nIm0ee9UgPOPEuWM6+gstQ9J+/L+TYRtnG2hvAMLqScbm4qcv5VfcFREvppEsrZPRvsIyWzXUW35Er55fSIzw9dcdz4QlgXGRPmgt8iJ/EM6f5djj5zB2qcMcx24D+7vbtq6iwTNVmG3oWgfZr4T8I68FwD4CjNPAXiAiDYCOCvdt5GZ70+P+0ra9o5h7Ahh+95pvOHTP22i65HApqf3gih5oH//X9ZnkfR8x74ZMyEomcD15esfxvc2PJ7tf3zn/nQxkZRKSZ/ppZPtwgO94dGdeOXHrs6Oa4uIS07SMlGhjPpkBLZkspX1vWuqg+WL25kyaLJNqLB2u9f5L1/chlnFKmbgqb3TOHTZosKxhm5wqQvJB0fCPuk8r7jjcbzyY1cLxY6xO+lLzlA2CXTX+e+b6RaKtt388NPZvZXVSDNFUJR/3rl/JstbmK6Nnd2Y7clhgi4yA+NkgRZKnG+H8yUrfYIruUiLG/m7cwVCUk83sWveBKKIMkdt7qW8pnZEmUNPjhV9UtJW9p/Z6TynTVVmaYrzPxrAdeL7I+k2ANjkbH+BrwMiugjARQCwevXqgYyIIsKaw5cNdOxCwJrDl+GlzzwMNz7wVDZ5aFTwwhMOwfOOWYHfe/kabHh0h7VvzeHL8PxjD8au/TM495TDsXzxBN573jNx3qlHgIjw7nPXYNNT+/CiEw/BD+563Drulaccjmvu2YZTj1qOF514CN569gl49pHLcfLhz8CuqQ7e/PPHI44Z92/bg7XHHoQVSybxuy89EW95yfE4aMkk3vzi4/D4zv143qoV+LnjDsZbzz4Bpx19IGa6jDeuXYVDli3CkQcuxhnHHIRdUzPYsn0/7t+2G885egWe3jsNAPjVM47GsYcswZ6pLl516uF4YOse/PpZq/HT+57ErqkZvPTkwzKb/+r1z8VxhyzFRIvw7COX41eedxQmWxFOOmwZrr5nK1516uFZ25c/6zD8bNN2dGPGa09PHrff+YUT8JP7tmVtTjnqQJz77KT/Xz9rNR7bsR/vfNlJWNSO8LZzTsTbzjkB39vwOE5YuRT/5QWrcc/ju3DfE7vx6I59OPWoA7N+fvOFx2LFkgnrdzlj9UF40QmH4gOveTbWPfQ0zl6zEssWtfHWs0/Apqf34llHLAcAvPKUI3D347sRM+M1px2JjU/sxiFLJ3HOM1finS87CccfuhRHrlgMAFixZDL7be/buicbwN70ouPwy887Cnc9tgsvPukQ/NkvnYL1m7bj3FMOx859Hdy/dTd2T3Ww5vBlOOmwZdixbwZveckJWLqojbOOPxgAcOwhS/BfX7Aa7YhwxPLFePGJh+LtLz0RZx13MP7o1c/EiSuXYfP2fTj1qOXZb7t4ooVXPPswbN87g0OWTuIvf+25+PxPHsT5px2Jt7xkO7ak9ykiwgkrl+LhJ/di30wXDz25F2ccc1B2r96w9hhMp5PiXnLSoQCAd5+7Bu0W4Y1rj8He6Q5WH7wUDz25p5CorgvUq+AXEV0J4AjPrg8w82Vpmx8C+ENmXpd+/ySA65j5i+n3zwD4j/S485j5t9PtvwngBcz8zjIb1q5dy+vWrat8UQqFQqEAiOgmZl7r29cz8mfmcwc452YAx4jvq9JtKNmuUCgUillCU2nlywFcSESLiOh4AGsA3ADgRgBriOh4IppEkhS+vCEbFAqFQhHAUJw/Ef0qgL8DsBLAt4loPTO/mpk3ENGlSBK5HQDvYOZuesw7AXwXQAvAJcy8YagrUCgUCkXf6Mn5zwco569QKBT9o4zz15LOCoVCMYZQ569QKBRjCHX+CoVCMYZQ569QKBRjiJFI+BLRVgAPDdHFoQC29Ww1+1C7+oPa1R/mq13A/LVtodl1LDOv9O0YCec/LIhoXSjjPZdQu/qD2tUf5qtdwPy1bZzsUtpHoVAoxhDq/BUKhWIMMS7O/+K5NiAAtas/qF39Yb7aBcxf28bGrrHg/BUKhUJhY1wif4VCoVAIqPNXKBSKMcSCdv7zabF4InqQiG4jovVEZBa9OZiIriCie9O/B/XqpyZbLiGiJ4jodrHNawsl+ER6D28lojNn2a4PEtHm9L6tJ6LXiH3vT+26m4he3aBdxxDRVUR0BxFtIKJ3pdvn9J6V2DWn94yIFhPRDUT0s9SuD6Xbjyei69PzfzUt64609PtX0+3XE9Fxs2zX54joAXG/Tk+3z9r//fR8LSK6hYj+Pf3e7P1i5gX5D0nJ6PsAnABgEsDPAJwyh/Y8COBQZ9tfAXhf+vl9AP5ylmw5G8CZAG7vZQuA1yBZhY0AvBDA9bNs1weRrBLntj0l/U0XATg+/a1bDdl1JIAz08/PAHBPev45vWclds3pPUuve1n6eQLA9el9uBTAhen2TwN4e/r5dwF8Ov18IYCvNnS/QnZ9DsDrPe1n7f9+er7/AeDLAP49/d7o/VrIkf9ZSBeLZ+ZpAGax+PmECwB8Pv38eQCvnY2TMvM1AJ6qaMsFAL7ACa4DsIKIjpxFu0K4AMBXmHmKmR8AsBHJb96EXVuY+eb08y4AdyJZk3pO71mJXSHMyj1Lr3t3+nUi/ccAXg7ga+l2936Z+/g1AK8gInsV9WbtCmHW/u8T0SoAvwjgn9LvhIbv10J2/kejuFh82YPRNBjA94joJkoWpweAw5l5S/r5MQCH+w+dFYRsmQ/38Z3pa/clghqbE7vSV+wzkESN8+aeOXYBc3zPUgpjPYAnAFyB5C1jOzN3POfO7Er37wBwyGzYxczmfn04vV9/Q0SLXLs8NteNjwN4L4A4/X4IGr5fC9n5zze8hJnPBHA+gHcQ0dlyJyfvcPNCdzufbAHwDwBOBHA6gC0APjpXhhDRMgBfB/BuZt4p983lPfPYNef3jJm7zHw6knW6zwLwrNm2wQfXLiI6DcD7kdj3cwAOBvDHs2kTEf0SgCeY+abZPO9Cdv5li8jPOph5c/r3CQDfRPJAPG5eI9O/T8yVfSW2zOl9ZObH0wc2BvCPyGmKWbWLiCaQONgvMfM30s1zfs98ds2Xe5bash3AVQBehIQ2MUvHynNndqX7DwTw5CzZdV5KnzEzTwH4LGb/fr0YwK8Q0YNI6OmXA/hbNHy/FrLznzeLxRPRUiJ6hvkM4FUAbk/t+a202W8BuGwu7EsRsuVyAG9KlQ8vBLBDUB2Nw+FYfxXJfTN2XZgqH44HsAbADQ3ZQAA+A+BOZv6Y2DWn9yxk11zfMyJaSUQr0s8HAHglknzEVQBenzZz75e5j68H8IP0TWo27LpLDOCEhFeX96vx35GZ38/Mq5j5OCR+6gfM/F/R9P2qM1s93/4hydbfg4Rv/MAc2nECEpXFzwBsMLYg4em+D+BeAFcCOHiW7PkXJHTADBIu8S0hW5AoHT6V3sPbAKydZbv+OT3vrel/+iNF+w+kdt0N4PwG7XoJEkrnVgDr03+vmet7VmLXnN4zAM8FcEt6/tsB/Jl4Dm5Akmj+VwCL0u2L0+8b0/0nzLJdP0jv1+0AvohcETRr//eFjS9FrvZp9H5peQeFQqEYQyxk2kehUCgUAajzVygUijGEOn+FQqEYQ6jzVygUijGEOn+FQqEYQ6jzVygUijGEOn+FQqEYQ/xf2g+2I8Iy68kAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "u2HaGRVEYGQS" + }, + "source": [ + "## Testing\n", + "The testing result will be the average reward of 5 testing" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 286 + }, + "id": "5yFuUKKRYH73", + "outputId": "3ad20d21-4c56-47b7-e617-d71e4211aed7" + }, + "source": [ + "fix(env, seed)\n", + "agent.network.eval() # set the network into evaluation mode\n", + "NUM_OF_TEST = 5 # Do not revise this !!!\n", + "test_total_reward = []\n", + "action_list = []\n", + "for i in range(NUM_OF_TEST):\n", + " actions = []\n", + " state = env.reset()\n", + "\n", + " img = plt.imshow(env.render(mode='rgb_array'))\n", + "\n", + " total_reward = 0\n", + "\n", + " done = False\n", + " while not done:\n", + " action, _ = agent.sample(state)\n", + " actions.append(action)\n", + " state, reward, done, _ = env.step(action)\n", + "\n", + " total_reward += reward\n", + "\n", + " img.set_data(env.render(mode='rgb_array'))\n", + " display.display(plt.gcf())\n", + " display.clear_output(wait=True)\n", + " \n", + " print(total_reward)\n", + " test_total_reward.append(total_reward)\n", + "\n", + " action_list.append(actions) # save the result of testing \n" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "-207.9114975585693\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAfbklEQVR4nO3deXRV9b338fc3AwmEMAdImBLaUGSQKUK01CKrXi1V4fqwFHutFvXSKq1Dr4reZ7WlvbUWa/Vely590Dq2FyuK1NV6L1pEqlUqoBEBUQaZwhAgTCFkOMn3+ePs2AiEjIeTnXxea+2VvX97ON9fzs4nO7/sc465OyIiEh4J8S5AREQaR8EtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIhE7PgNrOLzewTM9tkZnfF6nFERNobi8V93GaWCHwKXAjsBFYCV7n7+hZ/MBGRdiZWV9zjgU3uvsXdK4DngakxeiwRkXYlKUbH7QfsqLW8E5hQ18ZmppdvioicwN3tVO2xCu56mdksYFa8Hl9EJKxiFdyFwIBay/2Dts+5+3xgPuiKW0SkMWI1xr0SyDWzHDPrAMwAXonRY4mItCsxueJ294iZ/QBYAiQCT7r7ulg8lohIexOT2wEbXYSGSkRETlLXPyf1ykkRkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iETLM+c9LMtgJHgSog4u55ZtYD+AOQDWwFrnD3g80rU0REarTEFfcF7j7a3fOC5buApe6eCywNlkVEpIXEYqhkKvBMMP8MMC0GjyEi0m41N7gdeM3MVpvZrKCtj7vvDub3AH2a+RgiIlJLs8a4gYnuXmhmvYHXzWxD7ZXu7mbmp9oxCPpZp1onIiJ1M/dT5mrjD2Q2FygB/hWY5O67zSwTeNPdv1LPvi1ThIhIG+Ludqr2Jg+VmFmamaXXzAP/BKwFXgGuDTa7FvhjUx9DRERO1uQrbjMbDLwcLCYB/+3u95hZT+AFYCCwjejtgMX1HEtX3CIiJ6jrirvFhkqaQ8EtInKyFh8qERGR+FBwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGTqDW4ze9LMisxsba22Hmb2upltDL52D9rNzB4ys01mtsbMxsayeBGR9qghV9xPAxef0HYXsNTdc4GlwTLAN4HcYJoFPNoyZYqISI16g9vd/woUn9A8FXgmmH8GmFar/VmPWgF0M7PMlipWRESaPsbdx913B/N7gD7BfD9gR63tdgZtJzGzWWa2ysxWNbEGEZF2Kam5B3B3NzNvwn7zgfkATdlfRKS9auoV996aIZDga1HQXggMqLVd/6BNRERaSFOD+xXg2mD+WuCPtdqvCe4uyQcO1xpSERGRFmDupx+lMLMFwCSgF7AX+CmwGHgBGAhsA65w92IzM+BhonehlAIz3b3eMWwNlYiInMzd7VTt9Qb3maDgFhE5WV3BrVdOioiEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGTqDW4ze9LMisxsba22uWZWaGYFwTSl1rq7zWyTmX1iZhfFqnARkfaqIR8WfD5QAjzr7iOCtrlAibvff8K2w4AFwHggC/gLMMTdq+p5DH3mpIjICZr8mZPu/leguIGPMxV43t3L3f0zYBPREBcRkRbSnDHuH5jZmmAopXvQ1g/YUWubnUHbScxslpmtMrNVzahBRKTdaWpwPwp8CRgN7AZ+09gDuPt8d89z97wm1iAi0i41Kbjdfa+7V7l7NfA4/xgOKQQG1Nq0f9AmIiItpEnBbWaZtRb/Gai54+QVYIaZpZhZDpALvNe8EkVEpLak+jYwswXAJKCXme0EfgpMMrPRgANbge8BuPs6M3sBWA9EgNn13VEiIiKNU+/tgGekCN0OKCJykibfDigiIq2LgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZOoNbjMbYGbLzGy9ma0zs1uC9h5m9rqZbQy+dg/azcweMrNNZrbGzMbGuhMiIu1JQ664I8C/ufswIB+YbWbDgLuApe6eCywNlgG+SfTT3XOBWcCjLV61iEg7Vm9wu/tud38/mD8KfAz0A6YCzwSbPQNMC+anAs961Aqgm5lltnjlIiLtVKPGuM0sGxgD/B3o4+67g1V7gD7BfD9gR63ddgZtJx5rlpmtMrNVjaxZRKRda3Bwm1ln4CXgVnc/UnuduzvgjXlgd5/v7nnunteY/URE2rsGBbeZJRMN7d+7+6KgeW/NEEjwtShoLwQG1Nq9f9AmIiItoCF3lRjwW+Bjd3+g1qpXgGuD+WuBP9Zqvya4uyQfOFxrSEVERJrJoqMcp9nAbCLwFvARUB00/zvRce4XgIHANuAKdy8Ogv5h4GKgFJjp7qcdxzazRg2ziIi0B+5up2qvN7jPBAW3iMjJ6gpuvXJSRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIdOQDwseYGbLzGy9ma0zs1uC9rlmVmhmBcE0pdY+d5vZJjP7xMwuimUHRMLALDHeJUgb0pAPC84EMt39fTNLB1YD04ArgBJ3v/+E7YcBC4DxQBbwF2CIu1ed5jH0mZPSZmVnj6dv36GsWPEcoFNdGq6uz5xMasCOu4HdwfxRM/sY6HeaXaYCz7t7OfCZmW0iGuLvNrpqkZDLzh7PV8feRGJCEvu+tJnNm99B4S3N1agxbjPLBsYAfw+afmBma8zsSTPrHrT1A3bU2m0npw96EQB++cvvMW8ejBgBw4ZBVla8K2oOC0L7Rgb3mETPTrlk9h1GQsLpf+QmTZrE009/hSlTYPhwGDoUEjXKIieo94q7hpl1Bl4CbnX3I2b2KPAfRC8f/gP4DXBdI443C5jVuHKlLRs5cjCZmTB5cnR5925Yvz46/7//C5s2gTvs2QNVdQ68tQ79+o0kf8z1DO4xmdLK/Xy4/XlWv7+Q6urTF56RkcH48SUMHx5djkTgnXegshJ27oTFi6Pthw/D0aMx7oS0Wg0KbjNLJhrav3f3RQDuvrfW+seBPwWLhcCAWrv3D9q+wN3nA/OD/fW3o3zOglG9rKx/XHVfcEE0tKuqYMkSOH48Guy/+1386qxLSkpnvpQzkcz0UZRWHuCDbb9nxXtPc/z4oQYfo+Z7kJwMX/96dN4drr46Or92LXzySXT+2Wdh796TjyFtV0PuKjHgt8DH7v5ArfbMWpv9M7A2mH8FmGFmKWaWA+QC77VcydIeVVdHQzsSgdJSOHYsGt6tTUpKZ8af8x3GDv4OiQkd+HD7f7PivacoLT3Y7GPX/OKqqoKysuj34Nix6PdG2peGXHF/FfgO8JGZFQRt/w5cZWajiQ6VbAW+B+Du68zsBWA9EAFmn+6OEpHa3KMTRIcGCoIzbskS2LIluq64uHWGVYcOaZyT9y+MG3wNSQkpFGxfwLvvPdmk0K75PkQi8MYbUFEBhYXwyivR9SUlrfMXl5wZDbmr5G3gVLekvHqafe4B7mlGXdIOlZTAn/8cHf6oro6O4e7bF++qGq6qqoIjR/ZQVV3BgWOfsvmzv3LsWDHduw/g2LFiKiqONeg4BQXw+OOwbVv0+7B9e+v8RSXx0+B/TorE2vbtMHduvKtouqqqStatf5UkS6GkbB+HDu1i3Lgr6Nd3FLuL1rJy5YIGHeeBB2DVqhgXK6Gm4BZpQdXV1Wze9hZf/vL5DP/KFLK6jCU9JZPUpG7szFzD7t3r4l2itAEKbpEWYpZAbu75DB86hcz00fTsOIQES8LMyOkxiUMjtrF//xYqKzU4Lc2j4BZpAenpvcnOHk/e0O/Qv2v+54FdIzWpG5ndRjNw4Dg2b347jpVKW6DgFmmmzp0zOG/CDQzKOJe+nUeRYCf/WJkZfTuP4kuDz2P79tW66pZm0du6ijRRQkIiffoM5av5/8qoATPISh93ytCukZrUlaxuY8nOHn8Gq5S2SFfcIk2QkJDEsGEXMTz3EgZ1+yqdknvVu49ZAn06jyRnUD47drxPWZlesy5No+AWaaTOnXsxcsQljMz+P2Smj8Gs4X+4piZ1pV+PcQwcOI5PP30zdkVKm6ahEpFGysoayVmDvkVm+thGhTZEr7oz0s4ie8AEOnbsGqMKJewSEhJISqr7ulrBLdJIBw58xpGyXThNezljx6QeZPUcQ1bWyBauTMIuISGBQYMG8atf/YoRI0bUvd0ZrEmkTSgu3sGOPas5eHwL9X2C1KmYGRmdziJnUD6pqV1iUKGEUUJCArfeeisrV67k9ttvJ/E0b8Su4BZpJPcqNmx4nc1FyyiLNO1d/zol96Jfz7FkZQ1v4eokbJKSkpgyZQrLly/nF7/4BRkZGV94DcAp9zlDtYm0KYcP72bjZ8vo3W0og7pObMJYtzGg67mMHbGPo0f3sW/fphhVKq1ZXl4eDzzwAPn5+SQnJzd4P11xizTRtm0r2VX8PmWRw03aPykhhayuYxg8+NxGB7+E26BBg5g3bx6LFi3ia1/7WqNCG3TFLdJkFRWlbNn6LpndR5Pd7fwmhW+XlP4M6JPHZxkrKCraGIMqpbUwM/r06cMtt9zCzJkz6d27d71DInVRcIs0w7ZtK8nJib7UvWNy9/p3OEFSQip9u4wie9B49u//LAYVSmvQoUMHpkyZwuOPP06PHj3q/dDo+ujvM5FmqKwsY/Pmt9lTUtDoO0zcnUh1GZHq4yQlpTT56ktaLzPj4osvZvHixTz33HP06tWr2aENuuIWabYdOwrYlVNAn85n0ym5Z4P2qag6xsHjWyg8vJrthSv5+OPXqKqq/Hx9UlIKPXsOAozS0oMcPVoUo+olFsyMyZMnM2fOHM4991w6d+7cosevN7jNLBX4K5ASbP+iu/80+CDg54GewGrgO+5eYWYpwLPAOOAAcKW7b23RqkVakcrKUjZteZusHqPJ6X5BndtVexWllfspqdjDrkMfsH3XSrZseZdIpIJOnXqQltaTnJx8unQZSVqnCfTtORxwDhzZzFtvP055ud7bJAwGDhzIjBkz+NnPfkZqampMHqMhV9zlwGR3LzGzZOBtM/sf4EfAg+7+vJk9BlwPPBp8PejuXzazGcA84MqYVC/SSuza9RGFBz6gd9oI0jpkANGhkIqqo1R7NQeOf8Kxin3sKf6I4oPb2bp1JYcP7yI393y+lP01Oqf1Aowh/S/gcEfoUllGh8R0qr2S0spiEhP1x3Fr179/f2bOnMnVV19Nbm5uTIe+GvJhwQ6UBIvJweTAZODbQfszwFyiwT01mAd4EXjYzMyb8hIzkZAoLy9hy9Z36NdzDP0SJ3C0vJCjFbvZe3gtkepydu78kCNH9rBr1zrcqz7fr3v3AeRknE/3joMBp6r6CCUVxRQdK6ToyHoAiou3N/iDhuXMS01N5Y477uC6665j4MCBLTKGXZ8G/Ro3s0SiwyFfBh4BNgOH3D0SbLIT6BfM9wN2ALh7xMwOEx1O2V/X8TMzM4lEIuwL00d6S4t76623SEtLa9LLyFuDI0e2UlTyAaW+kwPFWyku3saOHe9TWVlGVVUE92o6dkz5wj4lJbvYe3wV+ysKAOfTt7Zy8NAhPvtsK9u2rQain2NZO+yldUhOTiY/P58bb7yRK6644rQvUW9pDQpuj541o82sG/AyMLS5D2xms4BZEB0TevbZZ/n2t7+t8G6HEhISuPzyy/nRj37E448/Hu9ymiU5OZWEhCQikQqqqirq3f7w4eMcO1ZFTk5fwKmoKMXdee655ygoGMzChQuprm7am1lJbJgZkyZN4sYbb+Siiy6iS5c4vN+MuzdqAn4C3EH0CjopaDsXWBLMLwHODeaTgu3sdMccN26cV1dX+5YtW/yGG25wokMxmtr41KFDB7/ssst8xYoVfvz4cZcvqqio8A0bNvgTTzzhU6dO9W7dusX9OWvvU8+ePX3x4sV+6NChmD//48aNc68rh+ta4f8I6gygWzDfEXgLuARYCMwI2h8DbgrmZwOPBfMzgBfqe4ygQHd3P3r0qM+ePduzsrLi/iRpis3UuXNnv+yyy/ydd97xysrKWJ//bUJ1dbUXFBT4iy++6BMnTvQJEyZ4UlJS3J/L9jJlZmb697//ff/oo4/O2HPe3OA+G/gAWAOsBX4StA8G3gM2EQ3xlKA9NVjeFKwfXN9j1A7uGu+9954PHDgw7k+Yppabunbt6lOnTvXly5d7dXV17M74dqCiosJfffVV//nPf+5nnXWWp6Wlxf35bYtTZmam33rrrb5mzZoz/hw3K7jPxHSq4K65wpg9e7auLEI+denSxadPn+5vvPGGV1VVteCpLdXV1R6JRPzPf/6z//rXv/b+/ft7enp63J/zsE89e/b0UaNG+fr16+N2zoYyuGtUVFT4HXfc4R07doz7k6mpcVNqaqrPmDHDly9f7uXl5S1wKsvpVFVVeUlJiS9fvtznzp3rM2fO9LS0NF34NHBKTk729PR0v+WWW3zjxo1eWloa178MQx3c7u5lZWX+/vvve25urptZ3J9gTaefEhMTffr06f7OO+/on45xVFZW5rt37/aHH37Yv/vd73rHjh3181PH+Tpp0iT/wx/+4Hv27Gk1/3cJfXDXKCws9F/+8pe6gmil04gRI3z69Om6S6QVikQi/umnn/qCBQt82rRpnpGREffzJd5Tzfj1hg0bzshdIo3VZoLbPTp0ct999/mQIUPi/sRrik4jRozwRx55xAsLC5tweko8rFu3zv/2t7/53Xff7eedd56npKTE/Tw6E1OHDh180qRJ/tprr/mGDRvi/TScVpsK7hrbt2/3CRMmxP1EaK+TmXlubq4/9thjXlRU1OjnT1qPSCTir7/+ut9///1+1llneZcuXeJ+frX0lJ2d7fPmzfMlS5Z4JBKJ97e8QU4X3ObuxFteXp6vWrWqUfu4Ozt37uSpp57i3nvvpaysLEbVSW2JiYkMHDiQ22+/nauvvpr09HS9j3Qb4e5UV1ezfPlytmzZwttvv81rr70GwIEDB6ioqP+VoK1Jeno6F1xwAZdeeimXXHIJffr0CdW5mpeXx6pVq05ZcGiDu0Z1dTVPPPEEt99+O0eP6m0vYyUxMZHs7Gx++MMfcv3115OWlhaqHwJpvMrKSioro+8RvnDhQrZv387SpUtZvXo1lZWVlJeXx7nCkyUkJDBgwACuu+46Jk6cyMSJE+nQoUO8y2qSNh3cAJFIhJ07dzJz5kzefPPNlitMMDNyc3O54YYbuO666+jRo4cCux07evQopaWlfPjhh7z88svs27ePxYsXAzVvhhWfPElKSmL69OmcffbZzJo1q02cp20+uGsUFRUxZ84cFi1axJEjR1qgsjMjPz+fzMzML7S9+eabHDx4ME4VRY0YMYKZM2dy1VVXnVSfCEBZWRnbtm0DoufskiVL2LNnD+++++4ZefwxY8YwZMgQ7rjjDkaOHBnaq+tTaTfBDdFxuoULF3LvvfdSUFDQIsdsaYMGDWLgwIEMHz6ca665hqFDh9K9+z8+aNbdWbduXZ1DP0899RQbNmw45bry8nJWrlzZrCufESNGcOONNzJt2jSysrKafBxpnw4ePMiGDRvYv38/v/nNb4hEIqxcubLFxsh79Ojx+c/OpZdeSp8+fVrkuK1NuwruGocOHeL6669n0aJFLXrcpsrJyWHQoEHcdtttDB06lCFDhsTkcY4fP86yZctO+VagZWVlzJs3j9LS0jr3v/nmm7n88svJyMiISX3S/lRVVbFs2TLKysrYvn07jzzyCFVVVWzcuLHBb1mbmJhIbm4u+fn53HTTTZxzzjkxrjr+2mVwAxw+fJjf/e53zJ07l/376/wch5jo2rUr3bp147bbbqNz585861vfonfv3phZ3Mbeam4lOp141idtX805ePz4cV566SUqKip45JFHKCoqIhKJUFT0xQ9F7tWrF1lZWdx5551Mnz6d5OTkM/IJM61Buw1uiJ4ob7zxBtdccw179uyJ2ZvSJycnk5KSwtVXX02/fv248MILOfvss0lNTVUQitTB3SkvL6e6upqDBw/yzDPPUFFRwfz587nyyiu56aabGDBgACkpKe3u56hdBzdET44DBw4wZ84cnnzyyRY7bmJiIp06dWLGjBnk5+dzySWX0L17d5KTk1vsMUTaG3enuLiYrl27kpTUfj8k+XTB3S6+K2ZGr169eOihh+jevTsLFixg165dTT7eN77xjc//gZeYmEhOTk67+fNNJNbMjJ49e8a7jFatXQR3jbS0NO6//34uv/xy7rzzTlasWEFVVf0fwjps2DCys7OZM2cOKSkpjBw5kk6dOp2BikVETtaugrvGeeedx9KlS/nxj3/Mgw8+SCQS+cL6Xr160bdvX2677TZ69+5NXl4effv2jVO1IiJf1C6DGyAlJYV77rmHsWPHct9997F371569erFzTffzKhRoxg7dqzusBCRVqne4DazVOCvQEqw/Yvu/lMzexr4OnA42PS77l5g0aT7L2AKUBq0vx+L4psrOTmZK6+8kmnTpkXfccusXf73WkTCpSFX3OXAZHcvMbNk4G0z+59g3R3u/uIJ238TyA2mCcCjwddWycxITU2NdxkiIg1W760QwVvDlgSLycF0unsIpwLPBvutALqZmd7oQkSkhTToHjYzSzSzAqAIeN3d/x6susfM1pjZg2aWErT1A3bU2n1n0CYiIi2gQcHt7lXuPhroD4w3sxHA3cBQ4BygBzCnMQ9sZrPMbJWZrdq3b18jyxYRab8a9aoRdz8ELAMudvfdwXBIOfAUMD7YrBAYUGu3/kHbicea7+557p6nNzQSEWm4eoPbzDLMrFsw3xG4ENhQM24d3EUyDVgb7PIKcI1F5QOH3X13TKoXEWmHGnJXSSbwjJklEg36F9z9T2b2hpllAAYUAN8Ptn+V6K2Am4jeDjiz5csWEWm/6g1ud18DjDlF++Q6tndgdvNLExGRU9E7I4mIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCxtw93jVgZkeBT+JdR4z0AvbHu4gYaKv9grbbN/UrXAa5e8apViSd6Urq8Im758W7iFgws1VtsW9ttV/QdvumfrUdGioREQkZBbeISMi0luCeH+8CYqit9q2t9gvabt/UrzaiVfxzUkREGq61XHGLiEgDxT24zexiM/vEzDaZ2V3xrqexzOxJMysys7W12nqY2etmtjH42j1oNzN7KOjrGjMbG7/KT8/MBpjZMjNbb2brzOyWoD3UfTOzVDN7z8w+DPr1s6A9x8z+HtT/BzPrELSnBMubgvXZ8ay/PmaWaGYfmNmfguW20q+tZvaRmRWY2aqgLdTnYnPENbjNLBF4BPgmMAy4ysyGxbOmJngauPiEtruApe6eCywNliHaz9xgmgU8eoZqbIoI8G/uPgzIB2YHz03Y+1YOTHb3UcBo4GIzywfmAQ+6+5eBg8D1wfbXAweD9geD7VqzW4CPay23lX4BXODuo2vd+hf2c7Hp3D1uE3AusKTW8t3A3fGsqYn9yAbW1lr+BMgM5jOJ3qcO8P+Aq061XWufgD8CF7alvgGdgPeBCURfwJEUtH9+XgJLgHOD+aRgO4t37XX0pz/RAJsM/AmwttCvoMatQK8T2trMudjYKd5DJf2AHbWWdwZtYdfH3XcH83uAPsF8KPsb/Bk9Bvg7baBvwXBCAVAEvA5sBg65eyTYpHbtn/crWH8Y6HlmK26w/wTuBKqD5Z60jX4BOPCama02s1lBW+jPxaZqLa+cbLPc3c0stLfumFln4CXgVnc/Ymafrwtr39y9ChhtZt2Al4GhcS6p2czsEqDI3Veb2aR41xMDE9290Mx6A6+b2YbaK8N6LjZVvK+4C4EBtZb7B21ht9fMMgGCr0VBe6j6a2bJREP79+6+KGhuE30DcPdDwDKiQwjdzKzmQqZ27Z/3K1jfFThwhkttiK8Cl5nZVuB5osMl/0X4+wWAuxcGX4uI/rIdTxs6Fxsr3sG9EsgN/vPdAZgBvBLnmlrCK8C1wfy1RMeHa9qvCf7rnQ8crvWnXqti0Uvr3wIfu/sDtVaFum9mlhFcaWNmHYmO239MNMCnB5ud2K+a/k4H3vBg4LQ1cfe73b2/u2cT/Tl6w93/hZD3C8DM0swsvWYe+CdgLSE/F5sl3oPswBTgU6LjjP833vU0of4FwG6gkuhY2vVExwqXAhuBvwA9gm2N6F00m4GPgLx413+afk0kOq64BigIpilh7xtwNvBB0K+1wE+C9sHAe8AmYCGQErSnBsubgvWD492HBvRxEvCnttKvoA8fBtO6mpwI+7nYnEmvnBQRCZl4D5WIiEgjKbhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCZn/D0FstkREZS2+AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Aex7mcKr0J01", + "outputId": "a36aaa35-ec20-4089-ddde-ab4742d3e90e" + }, + "source": [ + "print(np.mean(test_total_reward))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "-147.2620449863271\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "leyebGYRpqsF" + }, + "source": [ + "Action list" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "hGAH4YWDpp4u", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "24f547ee-6648-4d2a-f9c7-718b23e93251" + }, + "source": [ + "print(\"Action list looks like \", action_list)\n", + "print(\"Action list's shape looks like \", np.shape(action_list))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Action list looks like [[2, 2, 2, 2, 2, 2, 1, 2, 1, 2, 1, 2, 2, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 1, 2, 2, 2, 2, 2, 0, 3, 2, 3, 2, 3, 3, 2, 2, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 2, 3, 3, 2, 3, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 2, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 3, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 2, 1, 2, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 1, 2, 2, 2, 1, 1, 2, 2, 1, 1, 1, 2, 2, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 2, 2, 3, 2, 3, 3, 2, 3, 2, 3, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3], [2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 1, 1, 2, 2, 1, 2, 1, 2, 1, 2, 2, 2, 1, 2, 1, 1, 2, 1, 2, 2, 2, 2, 2, 2, 1, 1, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 3, 3, 2, 3, 3, 3, 3, 3, 2, 3, 2, 3, 3, 3, 3, 2, 2, 3, 3, 3, 3, 2, 2, 2, 3, 3, 3, 2, 2, 3, 3, 3, 3, 2, 2, 3, 3, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1, 2, 1, 2, 2, 1, 2, 1, 2, 2, 2, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 1, 2, 1, 2, 1, 1, 0, 2, 1, 1, 2, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 2, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 3, 3, 3, 3, 3, 2, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 2, 3, 3, 2, 2, 2, 3, 3, 2, 3, 0, 2, 3, 2, 0, 2, 3, 3, 2, 3, 2, 3, 2, 3, 2, 2, 3, 2, 3, 2, 2, 3, 3, 2, 3, 2, 2, 3, 2, 2, 2, 2, 3, 3, 2, 2, 2, 2, 3, 2, 2, 0, 1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 2, 1, 2, 2, 1, 1, 1, 1, 2, 2, 1, 1, 1, 2, 1, 2, 1, 2, 2, 1, 1, 2, 2, 1, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 1, 2, 2, 2, 0, 2, 3, 2, 3, 3, 3, 2, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 2, 2, 3, 2, 3, 3, 3, 2, 3, 2, 3, 2, 2, 3, 2, 3, 2, 3, 2, 3, 3, 2, 2, 3, 2, 3, 3, 3, 2, 3, 3, 2, 2, 3, 2, 3, 2, 2, 3, 3, 2, 2, 2, 2, 2, 2, 0, 2, 1, 2, 2, 0, 1, 2, 2, 0, 2, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 2, 2, 1, 2, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 2, 2, 2, 1, 2, 0, 2, 0, 3, 2, 3, 2, 0, 2, 0, 2, 3, 3, 3, 2, 2, 3, 2, 3, 2, 3, 2, 3, 3, 3, 2, 3, 3, 3, 3, 2, 2, 3, 2, 2, 2, 3, 2, 2, 3, 2, 3, 2, 2, 3, 2, 2, 3, 2, 0, 2, 3, 2, 2, 2, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 2, 2, 2, 2, 2, 2, 0, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 2, 1, 2, 2, 1, 2, 2, 1, 2, 1, 1, 2, 2, 0, 2, 1, 2, 1, 1, 1, 2, 1, 2, 2, 1, 2, 1, 2, 2, 2, 2, 1, 2, 2, 1, 2, 2, 1, 2, 1, 2, 2, 2, 1, 1, 2, 3, 3, 2, 3, 2, 3, 2, 3, 2, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 3, 2, 3, 3, 2, 2, 2, 2, 3, 2, 3, 2, 3, 3, 2, 3, 2, 2, 3, 2, 3, 2, 2, 2, 3, 2, 0, 2, 1, 2, 1, 2, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 1, 1, 2, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 2, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2, 1, 2, 2, 2, 1, 1, 2, 2, 2, 1, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2, 1, 1, 2, 2, 1, 2, 2, 2, 0, 2, 2, 1, 2, 2, 2, 3, 0, 3, 2, 3, 3, 2, 3, 3, 2, 3, 3, 2, 2, 3, 3, 3, 3, 2, 3, 3, 3, 2, 2, 3, 2, 2, 3, 3, 3, 2, 2, 2, 3, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 1, 2, 2, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 2, 2, 1, 2, 1, 1, 2, 2, 1, 2, 1, 1, 2, 2, 2, 1, 2, 1], [0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 1, 1, 2, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 2, 2, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 2, 3, 2, 3, 2, 3, 3, 2, 2, 3, 3, 2, 3, 2, 3, 3, 2, 3, 2, 3, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 2, 3, 2, 3, 2, 2, 2, 2, 2, 2, 0, 2, 1, 2, 2, 0, 1, 2, 1, 2, 2, 1, 1, 2, 0, 2, 1, 2, 2, 2, 2, 2, 1, 1, 2, 1, 0, 2, 2, 2, 2, 1, 2, 2, 2, 1, 1, 2, 2, 2, 1, 2, 1, 2, 2, 2, 2, 1, 2, 1, 2, 1, 1, 2, 2, 1, 2, 2, 2, 3, 2, 3, 2, 2, 3, 2, 3, 2, 3, 3, 2, 3, 3, 3, 2, 3, 3, 3, 3, 2, 3, 3, 2, 2, 2, 2, 3, 2, 3, 3, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 0, 0, 2, 2, 1, 1, 2, 2, 2, 1, 1, 2, 2, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 0, 1, 2, 1, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2, 1, 1, 2, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 2, 3, 3, 3, 2, 3, 3, 3, 2, 3, 3, 2, 2, 2, 3, 2, 3, 2, 2, 3, 3, 2, 3, 3, 3, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 2, 2, 1, 2, 1, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 3, 3, 2, 3, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 2, 1, 1, 1, 2, 1, 2, 1, 2, 2, 1, 2, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 2, 1, 2, 2, 1, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 1, 2, 2, 2, 3, 2, 2, 0, 2, 2, 2, 3, 2, 3, 2, 2, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 2, 3, 3, 3, 3, 2, 3, 3, 2, 3, 3, 3, 2, 2, 3, 2, 3, 3, 2, 3, 2, 3, 2, 2, 2, 3, 2, 3, 2, 3, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 2, 1, 1, 1, 2, 2, 1, 2, 1, 1, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]]\n", + "Action list's shape looks like (5,)\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n", + " return array(a, dtype, copy=False, order=order)\n" + ], + "name": "stderr" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fNkmwucrHMen" + }, + "source": [ + "Analysis of actions taken by agent" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "WHdAItjj1nxw", + "outputId": "20aea8b1-e011-4397-b775-7b4c4b593871" + }, + "source": [ + "distribution = {}\n", + "for actions in action_list:\n", + " for action in actions:\n", + " if action not in distribution.keys():\n", + " distribution[action] = 1\n", + " else:\n", + " distribution[action] += 1\n", + "print(distribution)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "{2: 1144, 1: 516, 0: 30, 3: 501}\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ricE0schY75M" + }, + "source": [ + "Saving the result of Model Testing\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "GZsMkGmIY42b", + "outputId": "c47a3123-eb1b-4dc1-f1b2-7a09a82cd8a6" + }, + "source": [ + "PATH = \"Action_List.npy\" # Can be modified into the name or path you want\n", + "np.save(PATH ,np.array(action_list)) " + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:2: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n", + " \n" + ], + "name": "stderr" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "asK7WfbkaLjt" + }, + "source": [ + "### This is the file you need to submit !!!\n", + "Download the testing result to your device\n", + "\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "c-CqyhHzaWAL", + "outputId": "38653c82-673e-4f90-8746-3a0424fe3aca" + }, + "source": [ + "from google.colab import files\n", + "files.download(PATH)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "\n", + " async function download(id, filename, size) {\n", + " if (!google.colab.kernel.accessAllowed) {\n", + " return;\n", + " }\n", + " const div = document.createElement('div');\n", + " const label = document.createElement('label');\n", + " label.textContent = `Downloading \"${filename}\": `;\n", + " div.appendChild(label);\n", + " const progress = document.createElement('progress');\n", + " progress.max = size;\n", + " div.appendChild(progress);\n", + " document.body.appendChild(div);\n", + "\n", + " const buffers = [];\n", + " let downloaded = 0;\n", + "\n", + " const channel = await google.colab.kernel.comms.open(id);\n", + " // Send a message to notify the kernel that we're ready.\n", + " channel.send({})\n", + "\n", + " for await (const message of channel.messages) {\n", + " // Send a message to notify the kernel that we're ready.\n", + " channel.send({})\n", + " if (message.buffers) {\n", + " for (const buffer of message.buffers) {\n", + " buffers.push(buffer);\n", + " downloaded += buffer.byteLength;\n", + " progress.value = downloaded;\n", + " }\n", + " }\n", + " }\n", + " const blob = new Blob(buffers, {type: 'application/binary'});\n", + " const a = document.createElement('a');\n", + " a.href = window.URL.createObjectURL(blob);\n", + " a.download = filename;\n", + " div.appendChild(a);\n", + " a.click();\n", + " div.remove();\n", + " }\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "download(\"download_899cef0d-01bc-40fd-a501-1573f2382641\", \"Action_List.npy\", 4689)" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "seT4NUmWmAZ1" + }, + "source": [ + "# Server \n", + "The code below simulate the environment on the judge server. Can be used for testing." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "U69c-YTxaw6b", + "outputId": "65dd4cf4-e667-469d-a7dc-34f6a053e1d0" + }, + "source": [ + "action_list = np.load(PATH,allow_pickle=True) # The action list you upload\n", + "seed = 543 # Do not revise this\n", + "fix(env, seed)\n", + "\n", + "agent.network.eval() # set network to evaluation mode\n", + "\n", + "test_total_reward = []\n", + "if len(action_list) != 5:\n", + " print(\"Wrong format of file !!!\")\n", + " exit(0)\n", + "for actions in action_list:\n", + " state = env.reset()\n", + " img = plt.imshow(env.render(mode='rgb_array'))\n", + "\n", + " total_reward = 0\n", + "\n", + " done = False\n", + "\n", + " for action in actions:\n", + " \n", + " state, reward, done, _ = env.step(action)\n", + " total_reward += reward\n", + " if done:\n", + " break\n", + "\n", + " print(f\"Your reward is : %.2f\"%total_reward)\n", + " test_total_reward.append(total_reward)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/torch/__init__.py:422: UserWarning: torch.set_deterministic is deprecated and will be removed in a future release. Please use torch.use_deterministic_algorithms instead\n", + " \"torch.set_deterministic is deprecated and will be removed in a future \"\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Your reward is : -29.53\n", + "Your reward is : -36.44\n", + "Your reward is : -194.16\n", + "Your reward is : -268.27\n", + "Your reward is : -207.91\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAc+klEQVR4nO3de3RU9b338feXJCRcwv1iCAmgDQcRFSFC9NAWWbVaq8JxsRR7qBbx0Cqtlx4ROc9qS/sstVqrz3HJ0getFts+qHgry9qDFpDWWpWLEQGxIuUWgSg3wZDLZL7PH7NDRyD3mUx25vNaa6/Z+7f3zP7+MptPNr/Zk23ujoiIhEenVBcgIiLNo+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQSVpwm9nFZvaBmW0xszuStR8RkXRjybiO28wygL8DFwK7gNXA1e6+KeE7ExFJM8k64x4HbHH3re5eDTwFTE7SvkRE0kpmkl43H9gZt7wLGF/fxmamr29KQnXt2pvMzByqKj+jqrqCnJzudO8ygKyMbq163dpoFZ9Xfcrnn+8jKyuHrl17E4nUUFGxH/dogqoXiXF3O1l7soK7UWY2C5iVqv1Lx9Wjxylc8JVbGN7vYjaVv8DyFfczfvy3KR46k9zsvFa99oGj/2D1R4+xZs3TnF8yk7OHTONo5ABvblzI+vVLE9QDkYYla6ikDCiIWx4ctB3j7gvdvdjdi5NUg6Sp006bQH7PsThRKioOEI3WJnwf1dUV7Cx7h4OV2+nXdThDBo8jN3dAwvcjcjLJCu7VQJGZDTOzzsA0QKcjknS9euVTOKiYXjlD2H34Xbb+429UVx9Nyr527lxH2f61RKKVDOo5hmHDSpKyH5HjJSW43T0CfB9YBrwPPOPuG5OxL5E6mZmd+dJpX2Fwr3Oprv2cj/etY9eu0qTtr7q6gq3/+Bt7Dr9Lny6nMiT/XHr0OCVp+xOpk7Qxbnd/GXg5Wa8vcrz8/LMYMexiemTns2H3s6x751lqa2uA2Oc7tV5DJFrZqn1EveYLyzt2rOXjYaWckjuaQb3GMGRIMe+991Kr9iHSmJR9OCmSSBkZWZx22gTyuo9m/9GtbC97i0OHdgdrnX37trGl+zKMk35IfxwDTn6hkxPl00+3HVuORKr4aOvr5PU+m/we4xiSP45t297m8OHy1nZJpF4KbukQCgrOYVDv0WR2yubjg++wffvaL6zftOkVPvrorwnZV1XVkS8s79xZyu5T3+WU7mfTs9tgCgvHsHHjMuoLf5HWUnBLh1BdXcHH+96lIrKPHR+v5tChj7+wPhqNcPTooaTsOxKpYuOmP9I1py+fHf44ONNXaEvyJOUr780uQl/AkQTIzu5Obm5/Dh7cTSTSurHslujZcxAVFQepqalo831Lx1TfF3AU3CIi7VR9wa0/6yoiEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIh06q/x21m24DDQC0QcfdiM+sDPA0MBbYBV7r7gdaVKSIidRJxxn2Bu4929+Jg+Q5gubsXAcuDZRERSZBkDJVMBhYF84uAKUnYh4hI2mptcDvwipmtNbNZQdtAd6+7S+seYGAr9yEiInFae8/JCe5eZmYDgFfNbHP8Snf3+u5uEwT9rJOtExGR+iXs1mVmNh84AvwHMNHdd5tZHvCau/9LI8/VrctERI6T8FuXmVk3M8utmwe+DmwAlgLXBptdC/y+pfsQEZETtfiM28xOBV4IFjOB/+fud5pZX+AZoBDYTuxywP2NvJbOuEVEjqO7vIuIhIzu8i4i0kEouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyjQa3mT1uZuVmtiGurY+ZvWpmHwaPvYN2M7MHzWyLma03szHJLF5EJB015Yz718DFx7XdASx39yJgebAM8A2gKJhmAQ8npkwREanTaHC7+5+B/cc1TwYWBfOLgClx7U96zJtALzPLS1SxIiLS8jHuge6+O5jfAwwM5vOBnXHb7QraTmBms8xsjZmtaWENIiJpKbO1L+DubmbeguctBBYCtOT5IiLpqqVn3HvrhkCCx/KgvQwoiNtucNAmIiIJ0tLgXgpcG8xfC/w+rv2a4OqSEuBQ3JCKiIgkgLk3PEphZouBiUA/YC/wE+BF4BmgENgOXOnu+83MgIeIXYVSAcxw90bHsDVUIiJyIne3k7U3GtxtQcEtInKi+oJb35wUEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyDQa3Gb2uJmVm9mGuLb5ZlZmZqXBdEncunlmtsXMPjCzi5JVuIhIumrKzYK/AhwBnnT3UUHbfOCIu9933LYjgcXAOGAQ8CdguLvXNrIP3XNSROQ4Lb7npLv/GdjfxP1MBp5y9yp3/wewhViIi4hIgrRmjPv7ZrY+GErpHbTlAzvjttkVtJ3AzGaZ2RozW9OKGkRE0k5Lg/th4DRgNLAb+GVzX8DdF7p7sbsXt7AGEZG01KLgdve97l7r7lHgUf45HFIGFMRtOjhoExGRBGlRcJtZXtzivwF1V5wsBaaZWbaZDQOKgLdbV6KIiMTLbGwDM1sMTAT6mdku4CfARDMbDTiwDfgugLtvNLNngE1ABJjd2BUlIiLSPI1eDtgmRehyQBGRE7T4ckAREWlfFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIdNocJtZgZmtNLNNZrbRzG4O2vuY2atm9mHw2DtoNzN70My2mNl6MxuT7E6IiKSTppxxR4D/dPeRQAkw28xGAncAy929CFgeLAN8g9jd3YuAWcDDCa9aRCSNNRrc7r7b3dcF84eB94F8YDKwKNhsETAlmJ8MPOkxbwK9zCwv4ZWLiKSpZo1xm9lQ4BzgLWCgu+8OVu0BBgbz+cDOuKftCtqOf61ZZrbGzNY0s2YRkbTW5OA2s+7Ac8At7v5Z/Dp3d8Cbs2N3X+juxe5e3JzniYikuyYFt5llEQvt37n780Hz3rohkOCxPGgvAwrinj44aBMRkQRoylUlBvwKeN/d749btRS4Npi/Fvh9XPs1wdUlJcChuCEVERFpJYuNcjSwgdkE4C/Ae0A0aP4vYuPczwCFwHbgSnffHwT9Q8DFQAUww90bHMc2s2YNs4iIpAN3t5O1NxrcbUHBLSJyovqCW9+cFBEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMg05WbBBWa20sw2mdlGM7s5aJ9vZmVmVhpMl8Q9Z56ZbTGzD8zsomR2QEQk3TTlZsF5QJ67rzOzXGAtMAW4Ejji7vcdt/1IYDEwDhgE/AkY7u61DexD95wUETlOi+856e673X1dMH8YeB/Ib+Apk4Gn3L3K3f8BbCEW4iIikgDNGuM2s6HAOcBbQdP3zWy9mT1uZr2DtnxgZ9zTdtFw0IsAcNdd3+Wee2DUKBg5EgYNSnVFbW/ixIn8+tf/wiWXwBlnwIgRkJGR6qqkvcls6oZm1h14DrjF3T8zs4eB/w148PhL4LpmvN4sYFbzypWO7MwzTyUvDyZNii3v3g2bNsXm/+d/YMsWcIc9e6C23oG3cOvfvz/jxh3hjDNiy5EIvPEG1NTArl3w4oux9kOH4PDh1NUpqdWk4DazLGKh/Tt3fx7A3ffGrX8UeClYLAMK4p4+OGj7AndfCCwMnq8xbjnGglG9QYP+edZ9wQWx0K6thWXL4OjRWLD/9repqzOZ6n4GWVnw1a/G5t1h+vTY/IYN8MEHsfknn4S9e098Dem4mnJViQG/At539/vj2vPiNvs3YEMwvxSYZmbZZjYMKALeTlzJko6i0VhoRyJQUQGffx4L73RS94urthYqK2M/g88/j/1sJL005Yz7X4FvA++ZWWnQ9l/A1WY2mthQyTbguwDuvtHMngE2ARFgdkNXlIjEc49NEBsaKA2OuGXLYOvW2Lr9+zt+WNX9HCIRWLECqquhrAyWLo2tP3Ik/X5xyT81Gtzu/jpwsktSXm7gOXcCd7aiLklDR47AH/4QG/6IRmNjuJ98kuqq2l5pKTz6KGzfHvs57NjR8X9RSfM0+cNJkWTbsQPmz091Fal3//2wZk2qq5D2TF95FxEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkdB23iEg706lTJzp1qv+8WmfcIiLtRKdOnRgyZAg///nPGTVqVP3btWFNIiJSj06dOnHLLbewevVqbrvtNjIa+EPsGioREUmhzMxMvv71rzNv3jzGjh1Lly5dGn9OG9QlIiInUVxczP33309JSQlZWVlNfp6CW0SkjQ0ZMoQbb7yRq6++moKCgsafcBwFt4hIGzAzBg4cyM0338yMGTMYMGAAZie9iXujFNwiIknWuXNnLrnkEh599FH69OnT4KV+TaHgFhFJEjPjoosu4qabbuLLX/4y3bt3T8jrKrhFRBLMzJg0aRJz587lvPPOS1hg12k0uM0sB/gzkB1s/6y7/yS4EfBTQF9gLfBtd682s2zgSWAssA+4yt23JbRqEZF2qrCwkGnTpvHTn/6UnJycpOyjKWfcVcAkdz9iZlnA62b2R+CHwAPu/pSZPQLMBB4OHg+4+5fMbBpwD3BVUqoXEWknBg8ezIwZM5g+fTpFRUUt/uCxKZpys2AHjgSLWcHkwCTgW0H7ImA+seCeHMwDPAs8ZGYWvI6ISIeSk5PDnDlzuO666ygsLGz1B49N0aQxbjPLIDYc8iVgAfARcNDdI8Emu4D8YD4f2Ang7hEzO0RsOOXT+l4/Ly+PSCTCJ+l4S2855i9/+QvdunUjnX/Hb9u2jUOHDpGRkUFtbW2qy5EGZGVlUVJSwg033MCVV17Z4FfUE61Jwe3utcBoM+sFvACMaO2OzWwWMAtiY0JPPvkk3/rWtxTeaahTp05cccUV/PCHP+TRRx9NdTntwm9+8xtKS0tZsmQJkUgkrX+ZtTdmxsSJE7nhhhu46KKL6NGjR9sX4e7NmoAfA3OInUFnBm3nAcuC+WXAecF8ZrCdNfSaY8eO9Wg06lu3bvXrr7/eiQ3FaOrgU+fOnf3yyy/3N998048ePeryRdXV1b5582Z/7LHHfPLkyd6rV6+Uv2fpPvXt29dffPFFP3jwYNLf/7Fjx7rXl8P1rfB/BnV/oFcw3wX4C3ApsASYFrQ/AtwYzM8GHgnmpwHPNLaPoEB3dz98+LDPnj3bBw0alPI3SVNypu7du/vll1/ub7zxhtfU1CT7+O8QotGol5aW+rPPPusTJkzw8ePHe2ZmZsrfy3SZ8vLy/Hvf+56/9957bfaetza4zwLeAdYDG4AfB+2nAm8DW4iFeHbQnhMsbwnWn9rYPuKDu87bb7/thYWFKX/DNCVu6tmzp0+ePNlXrVrl0Wg0eUd8GqiurvaXX37Zf/azn/npp5/u3bp1S/n72xGnvLw8v+WWW3z9+vVt/h63KrjbYjpZcNedYcyePVtnFiGfevTo4VOnTvUVK1Z4bW1tAg9tiUajHolE/A9/+IP/4he/8MGDB3tubm7K3/OwT3379vWzzz7bN23alLJjNpTBXae6utrnzJnjXbp0Sfmbqal5U05Ojk+bNs1XrVrlVVVVCTiUpSG1tbV+5MgRX7Vqlc+fP99nzJjh3bp104lPE6esrCzPzc31m2++2T/88EOvqKhI6f8MQx3c7u6VlZW+bt06LyoqcjNL+RusqeEpIyPDp06d6m+88YY+dEyhyspK3717tz/00EP+ne98x7t06aJ/P/UcrxMnTvSnn37a9+zZ024+dwl9cNcpKyvzu+66S2cQ7XQaNWqUT506VVeJtEORSMT//ve/++LFi33KlCnev3//lB8vqZ7qxq83b97cJleJNFeHCW732NDJvffe68OHD0/5G68pNo0aNcoXLFjgZWVlLTg8JRU2btzof/3rX33evHl+/vnne3Z2dsqPo7aYOnfu7BMnTvRXXnnFN2/enOq3oUEdKrjr7Nixw8ePH5/yAyFdJzPzoqIif+SRR7y8vLzZ75+0H5FIxF999VW/7777/PTTT/cePXqk/PhK9DR06FC/5557fNmyZR6JRFL9I2+ShoLb3J1UKy4u9jVr1jTrOe7Orl27eOKJJ7j77ruprKxMUnUSLyMjg8LCQm677TamT59Obm5uUv+YjrQddycajbJq1Sq2bt3K66+/ziuvvALAvn37qK6uTnGFzZObm8sFF1zAZZddxqWXXsrAgQNDdawWFxezZs2akxYc2uCuE41Geeyxx7jttts4fPhwgiuTOhkZGQwdOpQf/OAHzJw5k27duoXqH4E0X01NDTU1NQAsWbKEHTt2sHz5ctauXUtNTQ1VVVUprvBEnTp1oqCggOuuu44JEyYwYcIEOnfunOqyWqRDBzdAJBJh165dzJgxg9deey1xhQlmRlFREddffz3XXXcdffr0UWCnscOHD1NRUcG7777LCy+8wCeffMKLL74IxE6iUpUnmZmZTJ06lbPOOotZs2Z1iOO0wwd3nfLycubOncvzzz/PZ599loDK2kZJSQl5eXlfaHvttdc4cOBAiiqKGTVqFDNmzODqq68+oT4RgMrKSrZv3w7Ejtlly5axZ88e/va3v7XJ/s855xyGDx/OnDlzOPPMM0N7dn0yaRPcEBunW7JkCXfffTelpaUJec1EGzJkCIWFhZxxxhlcc801jBgxgt69ex9b7+5s3Lix3qGfJ554gs2bN590XVVVFatXr27Vmc+oUaO44YYbmDJlCoMGDWrx60h6OnDgAJs3b+bTTz/ll7/8JZFIhNWrVydsjLxPnz7H/u1cdtllDBw4MCGv296kVXDXOXjwIDNnzuT5559P6Ou21LBhwxgyZAi33norI0aMYPjw4UnZz9GjR1m5ciXRaPSEdZWVldxzzz1UVFTU+/ybbrqJK664gv79+yelPkk/tbW1rFy5ksrKSnbs2MGCBQuora3lww8/POlxejIZGRkUFRVRUlLCjTfeyLnnnpvkqlMvLYMb4NChQ/z2t79l/vz5fPppvfdxSIqePXvSq1cvbr31Vrp37843v/lNBgwYgJmlbOyt7lKihqSyPun46o7Bo0eP8txzz1FdXc2CBQsoLy8nEolQXl7+he379evHoEGDuP3225k6dSpZWVltcoeZ9iBtgxtiB8qKFSu45ppr2LNnT5N/wzdXVlYW2dnZTJ8+nfz8fC688ELOOusscnJyFIQi9XB3qqqqiEajHDhwgEWLFlFdXc3ChQu56qqruPHGGykoKCA7Ozvt/h2ldXBD7ODYt28fc+fO5fHHH0/Y62ZkZNC1a1emTZtGSUkJl156Kb179yYrKyth+xBJN+7O/v376dmzJ5mZTbpJV4fUUHCnxU/FzOjXrx8PPvggvXv3ZvHixXz88cctfr2vfe1rxz7Ay8jIYNiwYWnz3zeRZDMz+vbtm+oy2rW0CO463bp147777uOKK67g9ttv580332zSDVlHjhzJ0KFDmTt3LtnZ2Zx55pl07dq1DSoWETlRWgV3nfPPP5/ly5fzox/9iAceeIBIJPKF9f369eOUU07h1ltvZcCAARQXF3PKKaekqFoRkS9Ky+AGyM7O5s4772TMmDHce++97N27l379+nHTTTdx9tlnM2bMGF1hISLtUqPBbWY5wJ+B7GD7Z939J2b2a+CrwKFg0++4e6nFku6/gUuAiqB9XTKKb62srCyuuuoqpkyZEvuLW2Zp+em1iIRLU864q4BJ7n7EzLKA183sj8G6Oe7+7HHbfwMoCqbxwMPBY7tkZuTk5KS6DBGRJmv0UojgT8MeCRazgqmhawgnA08Gz3sT6GVm+kMXIiIJ0qRr2Mwsw8xKgXLgVXd/K1h1p5mtN7MHzCw7aMsHdsY9fVfQJiIiCdCk4Hb3WncfDQwGxpnZKGAeMAI4F+gDzG3Ojs1slpmtMbM1n3zySTPLFhFJX8361oi7HwRWAhe7++5gOKQKeAIYF2xWBhTEPW1w0Hb8ay1092J3L9YfNBIRabpGg9vM+ptZr2C+C3AhsLlu3Dq4imQKsCF4ylLgGospAQ65++6kVC8ikoaaclVJHrDIzDKIBf0z7v6Sma0ws/6AAaXA94LtXyZ2KeAWYpcDzkh82SIi6avR4Hb39cA5J2mfVM/2DsxufWkiInIy+stIIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZAxd091DZjZYeCDVNeRJP2AT1NdRBJ01H5Bx+2b+hUuQ9y9/8lWZLZ1JfX4wN2LU11EMpjZmo7Yt47aL+i4fVO/Og4NlYiIhIyCW0QkZNpLcC9MdQFJ1FH71lH7BR23b+pXB9EuPpwUEZGmay9n3CIi0kQpD24zu9jMPjCzLWZ2R6rraS4ze9zMys1sQ1xbHzN71cw+DB57B+1mZg8GfV1vZmNSV3nDzKzAzFaa2SYz22hmNwftoe6bmeWY2dtm9m7Qr58G7cPM7K2g/qfNrHPQnh0sbwnWD01l/Y0xswwze8fMXgqWO0q/tpnZe2ZWamZrgrZQH4utkdLgNrMMYAHwDWAkcLWZjUxlTS3wa+Di49ruAJa7exGwPFiGWD+LgmkW8HAb1dgSEeA/3X0kUALMDt6bsPetCpjk7mcDo4GLzawEuAd4wN2/BBwAZgbbzwQOBO0PBNu1ZzcD78ctd5R+AVzg7qPjLv0L+7HYcu6esgk4D1gWtzwPmJfKmlrYj6HAhrjlD4C8YD6P2HXqAP8XuPpk27X3Cfg9cGFH6hvQFVgHjCf2BY7MoP3YcQksA84L5jOD7SzVtdfTn8HEAmwS8BJgHaFfQY3bgH7HtXWYY7G5U6qHSvKBnXHLu4K2sBvo7ruD+T3AwGA+lP0N/ht9DvAWHaBvwXBCKVAOvAp8BBx090iwSXztx/oVrD8E9G3bipvs/wC3A9FguS8do18ADrxiZmvNbFbQFvpjsaXayzcnOyx3dzML7aU7ZtYdeA64xd0/M7Nj68LaN3evBUabWS/gBWBEiktqNTO7FCh397VmNjHV9STBBHcvM7MBwKtmtjl+ZViPxZZK9Rl3GVAQtzw4aAu7vWaWBxA8lgftoeqvmWURC+3fufvzQXOH6BuAux8EVhIbQuhlZnUnMvG1H+tXsL4nsK+NS22KfwUuN7NtwFPEhkv+m/D3CwB3Lwsey4n9sh1HBzoWmyvVwb0aKAo++e4MTAOWprimRFgKXBvMX0tsfLiu/ZrgU+8S4FDcf/XaFYudWv8KeN/d749bFeq+mVn/4EwbM+tCbNz+fWIBPjXY7Ph+1fV3KrDCg4HT9sTd57n7YHcfSuzf0Qp3/3dC3i8AM+tmZrl188DXgQ2E/FhslVQPsgOXAH8nNs74v1JdTwvqXwzsBmqIjaXNJDZWuBz4EPgT0CfY1ohdRfMR8B5QnOr6G+jXBGLjiuuB0mC6JOx9A84C3gn6tQH4cdB+KvA2sAVYAmQH7TnB8pZg/amp7kMT+jgReKmj9Cvow7vBtLEuJ8J+LLZm0jcnRURCJtVDJSIi0kwKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURC5v8Dix/mJUV1WnwAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TjFBWwQP1hVe" + }, + "source": [ + "# Your score" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "GpJpZz3Wbm0X", + "outputId": "7d4677c7-b285-42d3-d8c0-5f0a0d8230c8" + }, + "source": [ + "print(f\"Your final reward is : %.2f\"%np.mean(test_total_reward))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Your final reward is : -147.26\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wUBtYXG2eaqf" + }, + "source": [ + "## Reference\n", + "\n", + "Below are some useful tips for you to get high score.\n", + "\n", + "- [DRL Lecture 1: Policy Gradient (Review)](https://youtu.be/z95ZYgPgXOY)\n", + "- [ML Lecture 23-3: Reinforcement Learning (including Q-learning) start at 30:00](https://youtu.be/2-JNBzCq77c?t=1800)\n", + "- [Lecture 7: Policy Gradient, David Silver](http://www0.cs.ucl.ac.uk/staff/d.silver/web/Teaching_files/pg.pdf)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eZ7VDw-C19Qe" + }, + "source": [ + "" + ] + } + ] +} \ No newline at end of file diff --git a/11 Quantum ML/GuestLecture_QML.pdf b/11 Quantum ML/课件/GuestLecture_QML.pdf similarity index 100% rename from 11 Quantum ML/GuestLecture_QML.pdf rename to 11 Quantum ML/课件/GuestLecture_QML.pdf diff --git a/12 Life-Long&Compression/课件/life_v2.pdf b/12 Life-Long&Compression/课件/life_v2.pdf new file mode 100644 index 0000000..8b27fae Binary files /dev/null and b/12 Life-Long&Compression/课件/life_v2.pdf differ diff --git a/12 Life-Long&Compression/课件/life_v2.pptx b/12 Life-Long&Compression/课件/life_v2.pptx new file mode 100644 index 0000000..c6eecbd Binary files /dev/null and b/12 Life-Long&Compression/课件/life_v2.pptx differ diff --git a/12 Life-Long&Compression/课件/tiny_v6.pdf b/12 Life-Long&Compression/课件/tiny_v6.pdf new file mode 100644 index 0000000..5f9cf12 Binary files /dev/null and b/12 Life-Long&Compression/课件/tiny_v6.pdf differ diff --git a/12 Life-Long&Compression/课件/tiny_v6.pptx b/12 Life-Long&Compression/课件/tiny_v6.pptx new file mode 100644 index 0000000..b4bc9f4 Binary files /dev/null and b/12 Life-Long&Compression/课件/tiny_v6.pptx differ diff --git a/README.md b/README.md index 9956fae..49b0717 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,10 @@ 2021/05/21/ 更新RL 及 HW11 +2021/05/28/ 更新Quantum ML + +2021/06/04/ 更新Life-Long&Compression 及 HW12 + #------------------------------------------------------------------# B站视频地址：https://www.bilibili.com/video/BV1Wv411h7kN#reply4197445138 diff --git a/范例/HW12/HW12_EN.ipynb b/范例/HW12/HW12_EN.ipynb new file mode 100644 index 0000000..5694e58 --- /dev/null +++ b/范例/HW12/HW12_EN.ipynb @@ -0,0 +1,3092 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "hw12_reinforcement_learning_english_version.ipynb", + "provenance": [], + "collapsed_sections": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "de3a153737af485ea436d7e8393d8248": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_6345f8926212465291c04587353161f1", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_3aa84c8c097d4858a0c38f18dec8b060", + "IPY_MODEL_6647e68cf064416ca593b990bed81edf" + ] + } + }, + "6345f8926212465291c04587353161f1": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "3aa84c8c097d4858a0c38f18dec8b060": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_a25de7edbbee47cc8094f43125efa39b", + "_dom_classes": [], + "description": "Total: 86.3, Final: 0.0: 100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 400, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 400, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_a0bb5061ca0946ab89a89f9abadcebc8" + } + }, + "6647e68cf064416ca593b990bed81edf": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_d99d8e6bcbe0445eb7eddbfe31277635", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 400/400 [11:36<00:00, 1.74s/it]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_5af1c6579f0f4c2eb03df4063749569e" + } + }, + "a25de7edbbee47cc8094f43125efa39b": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "a0bb5061ca0946ab89a89f9abadcebc8": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "d99d8e6bcbe0445eb7eddbfe31277635": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "5af1c6579f0f4c2eb03df4063749569e": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + } + } + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Fp30SB4bxeQb" + }, + "source": [ + "# **Homework 12 - Reinforcement Learning**\n", + "\n", + "If you have any problem, e-mail us at ntu-ml-2021spring-ta@googlegroups.com\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yXsnCWPtWSNk" + }, + "source": [ + "## Preliminary work\n", + "\n", + "First, we need to install all necessary packages.\n", + "One of them, gym, builded by OpenAI, is a toolkit for developing Reinforcement Learning algorithm. Other packages are for visualization in colab." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "5e2bScpnkVbv", + "outputId": "52198e39-e2a2-4ea2-a1f3-4ba9545476d7" + }, + "source": [ + "!apt update\n", + "!apt install python-opengl xvfb -y\n", + "!pip install gym[box2d]==0.18.3 pyvirtualdisplay tqdm numpy==1.19.5 torch==1.8.1" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Hit:1 http://security.ubuntu.com/ubuntu bionic-security InRelease\n", + "Ign:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 InRelease\n", + "Hit:3 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease\n", + "Ign:4 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 InRelease\n", + "Hit:5 http://archive.ubuntu.com/ubuntu bionic InRelease\n", + "Hit:6 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 Release\n", + "Hit:7 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease\n", + "Hit:8 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 Release\n", + "Hit:9 http://archive.ubuntu.com/ubuntu bionic-updates InRelease\n", + "Hit:10 http://archive.ubuntu.com/ubuntu bionic-backports InRelease\n", + "Hit:11 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n", + "Hit:12 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease\n", + "Hit:13 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n", + "Reading package lists... Done\n", + "Building dependency tree \n", + "Reading state information... Done\n", + "86 packages can be upgraded. Run 'apt list --upgradable' to see them.\n", + "Reading package lists... Done\n", + "Building dependency tree \n", + "Reading state information... Done\n", + "python-opengl is already the newest version (3.1.0+dfsg-1).\n", + "xvfb is already the newest version (2:1.19.6-1ubuntu4.9).\n", + "The following package was automatically installed and is no longer required:\n", + " libnvidia-common-460\n", + "Use 'apt autoremove' to remove it.\n", + "0 upgraded, 0 newly installed, 0 to remove and 86 not upgraded.\n", + "Requirement already satisfied: gym[box2d] in /usr/local/lib/python3.7/dist-packages (0.17.3)\n", + "Requirement already satisfied: pyvirtualdisplay in /usr/local/lib/python3.7/dist-packages (2.1)\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (4.41.1)\n", + "Requirement already satisfied: numpy>=1.10.4 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.19.5)\n", + "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.4.1)\n", + "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.5.0)\n", + "Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.3.0)\n", + "Requirement already satisfied: box2d-py~=2.3.5; extra == \"box2d\" in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (2.3.8)\n", + "Requirement already satisfied: EasyProcess in /usr/local/lib/python3.7/dist-packages (from pyvirtualdisplay) (0.3)\n", + "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym[box2d]) (0.16.0)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "M_-i3cdoYsks" + }, + "source": [ + "\n", + "Next, set up virtual display，and import all necessaary packages." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "nl2nREINDLiw" + }, + "source": [ + "%%capture\n", + "from pyvirtualdisplay import Display\n", + "virtual_display = Display(visible=0, size=(1400, 900))\n", + "virtual_display.start()\n", + "\n", + "%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from IPython import display\n", + "\n", + "import numpy as np\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.optim as optim\n", + "import torch.nn.functional as F\n", + "from torch.distributions import Categorical\n", + "from tqdm.notebook import tqdm" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CaEJ8BUCpN9P" + }, + "source": [ + "# Warning ! Do not revise random seed !!!\n", + "# Your submission on JudgeBoi will not reproduce your result !!!\n", + "Make your HW result to be reproducible.\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "fV9i8i2YkRbO" + }, + "source": [ + "seed = 543 # Do not change this\n", + "def fix(env, seed):\n", + " env.seed(seed)\n", + " env.action_space.seed(seed)\n", + " torch.manual_seed(seed)\n", + " torch.cuda.manual_seed(seed)\n", + " torch.cuda.manual_seed_all(seed)\n", + " np.random.seed(seed)\n", + " random.seed(seed)\n", + " torch.set_deterministic(True)\n", + " torch.backends.cudnn.benchmark = False\n", + " torch.backends.cudnn.deterministic = True" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "He0XDx6bzjgC" + }, + "source": [ + "Last, call gym and build an [Lunar Lander](https://gym.openai.com/envs/LunarLander-v2/) environment." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "N_4-xJcbBt09" + }, + "source": [ + "%%capture\n", + "import gym\n", + "import random\n", + "env = gym.make('LunarLander-v2')\n", + "fix(env, seed) # fix the environment Do not revise this !!!" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NrkVvTrvWZ5H" + }, + "source": [ + "## What Lunar Lander？\n", + "\n", + "“LunarLander-v2”is to simulate the situation when the craft lands on the surface of the moon.\n", + "\n", + "This task is to enable the craft to land \"safely\" at the pad between the two yellow flags.\n", + "> Landing pad is always at coordinates (0,0).\n", + "> Coordinates are the first two numbers in state vector.\n", + "\n", + "![](https://gym.openai.com/assets/docs/aeloop-138c89d44114492fd02822303e6b4b07213010bb14ca5856d2d49d6b62d88e53.svg)\n", + "\n", + "\"LunarLander-v2\" actually includes \"Agent\" and \"Environment\". \n", + "\n", + "In this homework, we will utilize the function `step()` to control the action of \"Agent\". \n", + "\n", + "Then `step()` will return the observation/state and reward given by the \"Environment\"." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bIbp82sljvAt" + }, + "source": [ + "### Observation / State\n", + "\n", + "First, we can take a look at what an Observation / State looks like." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "rsXZra3N9R5T", + "outputId": "a36868de-bbbc-4de9-815b-0b43fc012c96" + }, + "source": [ + "print(env.observation_space)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Box(-inf, inf, (8,), float32)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ezdfoThbAQ49" + }, + "source": [ + "\n", + "`Box(8,)`means that observation is an 8-dim vector\n", + "### Action\n", + "\n", + "Actions can be taken by looks like" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "p1k4dIrBAaKi", + "outputId": "80c453ee-539f-4e40-c5d8-8e9dc8fffaef" + }, + "source": [ + "print(env.action_space)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Discrete(4)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dejXT6PHBrPn" + }, + "source": [ + "`Discrete(4)` implies that there are four kinds of actions can be taken by agent.\n", + "- 0 implies the agent will not take any actions\n", + "- 2 implies the agent will accelerate downward\n", + "- 1, 3 implies the agent will accelerate left and right\n", + "\n", + "Next, we will try to make the agent interact with the environment. \n", + "Before taking any actions, we recommend to call `reset()` function to reset the environment. Also, this function will return the initial state of the environment." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pi4OmrmZgnWA", + "outputId": "2635bdfb-a4dc-442b-a21a-f57af67edc4b" + }, + "source": [ + "initial_state = env.reset()\n", + "print(initial_state)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "[ 0.00396109 1.4083536 0.40119505 -0.11407257 -0.00458307 -0.09087662\n", + " 0. 0. ]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uBx0mEqqgxJ9" + }, + "source": [ + "Then, we try to get a random action from the agent's action space." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "vxkOEXRKgizt", + "outputId": "de93c740-f01c-464e-f436-a2b59e7dc7e5" + }, + "source": [ + "random_action = env.action_space.sample()\n", + "print(random_action)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mns-bO01g0-J" + }, + "source": [ + "More, we can utilize `step()` to make agent act according to the randomly-selected `random_action`.\n", + "The `step()` function will return four values:\n", + "- observation / state\n", + "- reward\n", + "- done (True/ False)\n", + "- Other information" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "E_WViSxGgIk9" + }, + "source": [ + "observation, reward, done, info = env.step(random_action)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "yK7r126kuCNp", + "outputId": "2f3363d9-5bc3-4ba5-86f2-1c4abc89f179" + }, + "source": [ + "print(done)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "False\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GKdS8vOihxhc" + }, + "source": [ + "### Reward\n", + "\n", + "\n", + "> Landing pad is always at coordinates (0,0). Coordinates are the first two numbers in state vector. Reward for moving from the top of the screen to landing pad and zero speed is about 100..140 points. If lander moves away from landing pad it loses reward back. Episode finishes if the lander crashes or comes to rest, receiving additional -100 or +100 points. Each leg ground contact is +10. Firing main engine is -0.3 points each frame. Solved is 200 points. " + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "vxQNs77hi0_7", + "outputId": "4633d678-be4f-4f52-8f91-1b6681642580" + }, + "source": [ + "print(reward)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "-0.8588900517154912\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Mhqp6D-XgHpe" + }, + "source": [ + "### Random Agent\n", + "In the end, before we start training, we can see whether a random agent can successfully land the moon or not." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 269 + }, + "id": "Y3G0bxoccelv", + "outputId": "11ad28c1-058b-4243-bf35-1fdfbb60be9e" + }, + "source": [ + "env.reset()\n", + "\n", + "img = plt.imshow(env.render(mode='rgb_array'))\n", + "\n", + "done = False\n", + "while not done:\n", + " action = env.action_space.sample()\n", + " observation, reward, done, _ = env.step(action)\n", + "\n", + " img.set_data(env.render(mode='rgb_array'))\n", + " display.display(plt.gcf())\n", + " display.clear_output(wait=True)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAd2ElEQVR4nO3de3RW9b3n8fc39xCu4RLC3SCtRYvcL8taKD0cKWtmsC0qTr3AaCnWarvmTOfomTVHzzmrx1W0doZlh5baKrQVag9eWBZQBI8VKSAochWIIVxyAgECgYDkxnf+eHbCIwnk9iRPdvJ5rfWs7P3bez/7+wvP82Hn9+z9bHN3REQkPBLiXYCIiDSOgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREKmxYLbzKab2T4zyzWzx1pqPyIiHY21xHncZpYI7AemAUeBD4C73X1PzHcmItLBtNQR93gg193z3L0cWA7MbKF9iYh0KEkt9Lz9gSNR80eBCVdb2cx0+aaIyBXc3epqb6ngrpeZzQPmxWv/IiJh1VLBXQAMjJofELTVcPfFwGLQEbeISGO01Bj3B8AwM7vOzFKA2cDKFtqXiEiH0iJH3O5eaWY/AN4EEoHfuvvultiXiEhH0yKnAza6CA2ViIjUcrUPJ3XlpIhIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREKmWfecNLN84BxQBVS6+1gzywT+CAwB8oE73f1088oUEZFqsTji/pq7j3T3scH8Y8A6dx8GrAvmRUQkRlpiqGQmsCSYXgLc3gL7EBHpsJob3A68ZWbbzGxe0Jbl7oXB9DEgq5n7EBGRKM0a4wa+4u4FZtYHWGtmn0QvdHc3M69rwyDo59W1TERErs7c68zVxj+R2ZNAKfBdYIq7F5pZNvDv7v7FeraNTREiIu2Iu1td7U0eKjGzDDPrUj0N/C2wC1gJ3B+sdj/welP3ISIitTX5iNvMcoBXg9kk4CV3/4mZ9QReBgYBh4icDlhcz3PpiFtE5ApXO+KO2VBJcyi4RURqi/lQiYiIxIeCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiFTb3Cb2W/NrMjMdkW1ZZrZWjM7EPzsEbSbmS00s1wz22Fmo1uyeBGRjqghR9wvAtOvaHsMWOfuw4B1wTzAN4BhwWMesCg2ZYqISLV6g9vd/wIUX9E8E1gSTC8Bbo9qX+oRm4DuZpYdq2JFRKTpY9xZ7l4YTB8DsoLp/sCRqPWOBm21mNk8M9tqZlubWIOISIeU1NwncHc3M2/CdouBxQBN2V5EpKNq6hH38eohkOBnUdBeAAyMWm9A0CYiIjHS1OBeCdwfTN8PvB7Vfl9wdslEoCRqSEVERGLA3K89SmFmy4ApQC/gOPAE8BrwMjAIOATc6e7FZmbAc0TOQrkAzHX3esewNVQiIlKbu1td7fUGd2tQcIuI1Ha14NaVkyIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQmZeoPbzH5rZkVmtiuq7UkzKzCz7cFjRtSyx80s18z2mdltLVW4iEhH1ZCbBX8VKAWWuvtNQduTQKm7P3PFusOBZcB4oB/wNvAFd6+qZx+656SIyBWafM9Jd/8LUNzA/cwElrt7mbsfBHKJhLiIiMRIc8a4f2BmO4KhlB5BW3/gSNQ6R4O2WsxsnpltNbOtzahBRKTDaWpwLwKGAiOBQuBnjX0Cd1/s7mPdfWwTaxAR6ZCaFNzuftzdq9z9EvBrLg+HFAADo1YdELSJiEiMNCm4zSw7avabQPUZJyuB2WaWambXAcOALc0rUUREoiXVt4KZLQOmAL3M7CjwBDDFzEYCDuQD3wNw991m9jKwB6gEHq7vjBIREWmcek8HbJUidDqgiEgtTT4dUERE2hYFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjI1BvcZjbQzN4xsz1mttvMfhi0Z5rZWjM7EPzsEbSbmS00s1wz22Fmo1u6EyIiHUlDjrgrgb9z9+HAROBhMxsOPAasc/dhwLpgHuAbRO7uPgyYByyKedUiIh1YvcHt7oXu/mEwfQ7YC/QHZgJLgtWWALcH0zOBpR6xCehuZtkxr1xEpINq1Bi3mQ0BRgGbgSx3LwwWHQOygun+wJGozY4GbVc+1zwz22pmWxtZs4hIh9bg4DazzsAK4EfufjZ6mbs74I3Zsbsvdvex7j62MduJiHR0DQpuM0smEtp/cPdXgubj1UMgwc+ioL0AGBi1+YCgTUREYqAhZ5UY8Btgr7s/G7VoJXB/MH0/8HpU+33B2SUTgZKoIRUREWkmi4xyXGMFs68A7wE7gUtB8z8QGed+GRgEHALudPfiIOifA6YDF4C57n7NcWwza9Qwi4hIR+DuVld7vcHdGhTcIiK1XS24deWkiEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQqYhNwseaGbvmNkeM9ttZj8M2p80swIz2x48ZkRt87iZ5ZrZPjO7rSU7ICLS0TTkZsHZQLa7f2hmXYBtwO3AnUCpuz9zxfrDgWXAeKAf8DbwBXevusY+dM9JEZErNPmek+5e6O4fBtPngL1A/2tsMhNY7u5l7n4QyCUS4iIiEgONGuM2syHAKGBz0PQDM9thZr81sx5BW3/gSNRmR7l20IsA8K//+j1++lO46SYYPhz69Yt3Ra1vypQpvPjiF5kxA268EW64ARIT412VtDVJDV3RzDoDK4AfuftZM1sE/Avgwc+fAf+tEc83D5jXuHKlPfvyl3PIzoapUyPzhYWwZ09kes0ayM0Fdzh2DKquOvAWbr1792b8+FJuvDEyX1kJGzdCRQUcPQqvvRZpLymBc+fiV6fEV4OC28ySiYT2H9z9FQB3Px61/NfAG8FsATAwavMBQdvnuPtiYHGwvca4pYYFo3r9+l0+6v7a1yKhXVUFb74Jn30WCfbf/z5+dbak6t9BcjJMnhyZdod77olM79oF+/ZFppcuhePHaz+HtF8NOavEgN8Ae9392aj27KjVvgnsCqZXArPNLNXMrgOGAVtiV7J0RJcuRUK7shIuXIDz5yPh3ZFU/8dVVQUXL0Z+B+fPR3430rE05Ij7FuBeYKeZbQ/a/gG428xGEhkqyQe+B+Duu83sZWAPUAk8fK0zSkSiuUceEBka2B684t58E/LyIsuKi9t/WFX/HiorYf16KC+HggJYuTKyvLS04/3HJZfVG9zuvgGo65SUVdfY5ifAT5pRl3RApaXw5z9Hhj8uXYqM4Z44Ee+qWt/27fDrX8OhQ5Hfw+HD7f8/KmmcBn84KdLSDh+GJ5+MdxXx9+yzsHVrvKuQtkyXvIuIhIyCW0QkZDRUIhITRkJC5EoZ9yrq+yoJkeZQcIvEwLBhX2Xw4LEAHDu2l5KSQgAuXDjDqVMH41matEMKbpFmMjP69x/Bl/rOpFNyJqd7H+R8eeR0mM/KT3Pq7KcAVFSUkZf3PpeCU0SKiw9TWXkxbnVLeCm4RZopK+uL9On2RbqnDSIxIYX05MyaZRVVF7mYeRqAKi9nUN9xuEeC+/ipPZRVnAfg8OGtFBTsbP3iJZQU3CLN1KVLHzql9iYxIaXWsuTENJITL19k3C11UM10/67jqbpURtH5XRQVHWiVWqV9UHCLNENiYgpDhkwgK+PLDVrf7PK1bF1TI1/EcuZifkuUJu2YTgcUaYYePQbSNb0fyYlp8S5FOhAFt8hVdE1KYnT37jXf99A5LY2BPXuSknT5D9WePQfTNa0/SQkKbmk9GioRqUPXpCS+m5PDdRkZ/LmwkLVFRYzOyaFrejonz53jw7w8SEhh0MAx9MkYHu9ypYPREbdIHbLS0hjUqRPJCQnc3L07iWY1p/FVVUUusBk8eCz9e4wmNalbTPfdtWsWSUk6gperU3CL1OFAaSk7S0o4cO4cv87Lo8KNqi4j8B5jOXz2EhVVVaSndyclqQsJFrt7i6WkZDBu9HcYNfLbJCWlxux5pX3RUEkb0L17d1JTUyktLeX8+fPxLkcCvzkYueLRgb59v8SAfrfSK+MLVI3sxYa/LiYxMZkEi91byCyBnJyJDEj7GwqGVFB0Yh+HDulrAqU2BXecJCUlceONNzJz5ky+9a1vMXToUDZs2MD777/PCy+8QGFhYc2f5hIf1d82kpycRs6QSWR3HYmRQEnpfwAwqP8Yene6IXb7c+fUqYN061XAoG6TGHljIefOnaC4+FDM9iHtg4K7lWVkZHDHHXcwceJE7rjjDjIzL19lN336dG677TYeeeQRFi1axM6dO3n11VcV4HE2bNhkhg/+z6Ql9WBXwcvs3r0aALNEzGI52uicOJFH1077MR9Fv+6jGTJkPGfOFHDpUmUM9yNhp+BuJZmZmXzzm9/kxz/+McOGDSMhoe43vJnRp08fnnjiCcrKyti+fTsLFizgwIED7NypS6JbW58+X+BLOdPp0/lGjpXu4NNDGzh79hidO/dusX3u2b+ajKSzDOs5nS8P/RZnzx4nN/cvLbY/CZ96g9vM0oC/AKnB+v/m7k8ENwJeDvQEtgH3unu5maUCS4ExwCngLnfPb6H627SEhATGjBnD448/ztChQxkxYkSjtk9NTWXChAmsWLGCwsJCXnvtNX73u9/xySefcPr06RaqWqolJaUwbNhkBmZO4JJXUnD6Qw4d2gZAnz7XU+kXOXBqNX07jyIxIfJWSkvqQVJC8z5UPHEil/8ov0hm+nX07TyCG4b+DSdP5nHmzNFm90nah4YccZcBU9291MySgQ1mthr478DP3X25mf0SeABYFPw87e7Xm9ls4KfAXS1Uf5vUv39/7rvvPiZNmsS0adNIS2v+qV3Z2dnMnz+f+fPns2nTJn7xi1+wZcsW8vLyqKrSvZhbQq9eOQzMGkPnlL7sKXqND7b+gYsXSwA4dGgb588Xk5CQxNCht9AlNZ1eaal4ykBI7AxARnIW3dOGAJBgCaQkdvncJe8AZZXnOFX6KceP76tpq6i4yN5P1tK3z5cY0e87DOn9VU584QAfbF2mIRMBGnazYAdKg9nk4OHAVOC/Bu1LgCeJBPfMYBrg34DnzMy8nX+zfHp6OuPHj+fuu+9m2rRpDBky5KrDIU1V/aafNGkSEyZMoLy8nGXLlvH0009z7NgxHYXHUEZGL0aNmMWArhM4eX4fBw9v4MyZgprlVVXlNWFbWLiHvunpjOmZSb5155xFjri7d+9PVtYXAUhOTKdP18iFOobRJ+MmkhJSKas6x2flZ7hwofhz+z9//hRbP1pORlof+mR8idTULiQmpii4BWjgGLeZJRIZDrke+AXwKXDG3atfRUeB/sF0f+AIgLtXmlkJkeGUkzGsu00wM3r16sV3v/tdxowZw4wZM2JydN0QCQkJpKWlMWfOHO666y4OHTrEc889x5IlSygvL6eioqJV6oiVtLQ0Nm7cSEZGRhu5e8xFjp3YTueuXbhYdpa8g/9OWloKUPsbAAHOAu8UFwOXA7i4eD/5+ZGx6ZSUTgwYMBKIvG4GDxpHcnI6AGlpyWRkZFBVVUl+fj4lJSV06tSJM2cOs333y/ToMZBDh7dSUXGhJTssIWKNeZOYWXfgVeB/Ay+6+/VB+0BgtbvfZGa7gOnufjRY9ikwwd1PXvFc84B5weyYZvekFSUkJJCVlcWcOXN49NFH6dOnT8yPrpuioqKCU6dOsXbtWn71q1/x17/+tc2dkVL9e+ratSuzZs2qmf/+979PVlZWPEurxSyRlJR03J3y8tieX5+S0qnmjBT3S5SX1w7lV155hY8//pjExBSqqsp57bXXOHny8tuorf3bSuy5u9XV3qjgBjCzfwQ+A/4e6BscVU8CnnT328zszWD6r2aWBBwDel9rqCQ9Pd0vXmz7dwJJTExk2rRpPPbYYwwZMoTBgwfHu6SrKi4upqioiOeee44//elPFBUVxaWOTp068fWvf52EhATMjEcffZTs7GySk5PJycmpNeYrdXN3Dh06RPX75OLFiyxYsIALFy4H/rZt2zh6VB9gtidNDm4z6w1UuPsZM0sH3iLygeP9wIqoDyd3uPv/M7OHgS+7+/zgw8lvufud19rHzTff7IsWLaqZLy4u5umnn6750O3MmTPs3r27wZ2NtRtuuIEHH3yQW265hdGjR5OSUvefy23Vnj17yM/P56mnnmLTpk1UVsZ+nDQ1NZUxY8ZgZowePZrZs2fXtI8aNapN/EXS3u3fv7/miNzdWbhwIQUFl8flCwoKyM/Pj1N10hTNCe4RRD58TCTy3SYvu/s/m1kOkdMBM4GPgHvcvSw4ffB3wCgiA36z3T3vWvsYO3asb9169Ut7i4qK2LRpU818ZWUlzzzzDGfOnKlpi/WHc926dWPq1KnMmTOHcePGkZ2dXf9GbVxFRQVvv/02W7du5fnnn+fw4cONfo5+/frRrVvkS5VGjhxZE9CdOnVi6tSpCug2LDc3lz179tTML1u2jI8//rhmPi8vj7KysniUJlcRs6GSllBfcF/J3WuN723atOlzL8oXXniBgwcv3137s88+o6Sk5JrPm5iYSO/evZk8eTIPPfQQt956a7sMInenoKCA1atX88orr/Dee+/V+o6Ubt26kZ4e+fBsxIgRzJo1C4ApU6aQk5MDRD5ka4+/n47i0qVLn/sg+PXXX+fUqVMArFmzho0bNzbofSMtp10Fd0NcvHjxc+c37969m9WrV9fMf/bZZyxevJjy8nIgcp70I488wty5c0lNTQ3dcEhTlZWVsX79erZs2fK59hkzZjB8eOT0taSkJFJT9U11HUn1mUl79uxh1apV/PGPf+Tw4cOUlZW1yFCb1K3DBXd9Ll26xMmTJ2uO3JOTk8nMzNSHZSJ1KC4upry8nNWrV7N582YOHDjAu+++q4u/WpiCW0RipqSkhGPHjrF48WLy8vLYsWMHeXnX/ChLmkDBLSIt5uDBgxw5coQFCxZQUlLCjh07OHv2bLzLCj0Ft4i0mo0bN7Jv3z6effZZKisryc3N1dh4Eyi4RaRVVZ/9VVVVxYoVK/joo4/4/e9/T2VlJSdOnIh3eaGg4BaRuKqqqqKsrIzTp0/z4osv8v777/Pee+9RWVlJGK6cjgcFt4i0KefPn6e0tJT9+/fz0ksvsXbtWvLz82udX96RXS24dQccEYmLjIwMMjIyyMrK4tZbb6WgoIDS0lLWr1/PW2+9xbp16zh37ly8y2yTdMQtIm2Ou7Njxw5+9rOfsWrVqporOjuaqx1x63plEWlzzIybb76ZpUuXsnr1aubOnUvv3i13n8+wUXCLSJs2btw4nn/+edasWcNDDz1EZmZmvEuKOwW3iLR5CQkJjB49moULF/Luu+8yf/58OnXqFO+y4kbBLSKhkZSUxE033cTChQv54IMPmDt3bqvdLrAtUXCLSOgkJyczfPhwFi9ezLZt27j33nvp3LlzvMtqNQpuEQmtpKQkhg8fzpIlS9iwYQP33ntvhxgDV3CLSOhFn4WyZs0aXnrpJUaMGEFiYmK8S2sRCm4RaVfGjRvH7Nmz2bJlC0uXLmXEiBHt7sYo9Qa3maWZ2RYz+9jMdpvZPwXtL5rZQTPbHjxGBu1mZgvNLNfMdpjZ6JbuhIhINDMjNTWVu+++m82bN/PLX/6SUaNGkZycHO/SYqIhR9xlwFR3vxkYCUw3s4nBsh+7+8jgsT1o+wYwLHjMAxbVekYRkVZgZqSlpTF37lzeeecdFi1axPjx40N/r9R6q/eI0mA2OXhc6zr5mcDSYLtNQHczC/8t0kUk1Lp168YDDzzAqlWrWLJkCZMmTYp3SU3WoP92zCzRzLYDRcBad98cLPpJMBzyczOrvptsf+BI1OZHgzYRkbjr2bMn99xzD2+88QbLly9n3Lhx8S6pRo8ePZg8eTKTJ0++5umNDfp2QHevAkaaWXfgVTO7CXgcOAakAIuBvwf+uaEFmtk8IkMpDBo0qKGbiYjERGZmJnfddRczZsxg1apVPPXUU+zatSumN0DOyMggJyenVvv111/Pgw8+WKu9V69ejB8/HoCxY8de9Xkb9bWu7n7GzN4Bprv7M0FzmZm9APyPYL4AGBi12YCg7crnWkwk8Bk7dmz8v6JQRDqkLl26cOedd3L77bezYsUKFixYwN69eykvL6+1bmpqap1fdnXdddfxwAMP1Grv27cv06ZNq9VuZpjV+cV/DVJvcJtZb6AiCO10YBrwUzPLdvdCi+z9dmBXsMlK4AdmthyYAJS4e2GTKxQRaWHRZ6F8+9vfZtmyZRw7dqzWegMGDGDWrFm12hMSElr1lMOGHHFnA0vMLJHImPjL7v6Gma0PQt2A7cD8YP1VwAwgF7gAzI192SIisVcd4HPmzIl3KddUb3C7+w5gVB3tU6+yvgMPN780ERGpS7hPZhQR6YAU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMubu8a4BMzsH7It3HS2kF3Ay3kW0gPbaL2i/fVO/wmWwu/eua0FSa1dyFfvcfWy8i2gJZra1PfatvfYL2m/f1K/2Q0MlIiIho+AWEQmZthLci+NdQAtqr31rr/2C9ts39audaBMfToqISMO1lSNuERFpoLgHt5lNN7N9ZpZrZo/Fu57GMrPfmlmRme2Kass0s7VmdiD42SNoNzNbGPR1h5mNjl/l12ZmA83sHTPbY2a7zeyHQXuo+2ZmaWa2xcw+Dvr1T0H7dWa2Oaj/j2aWErSnBvO5wfIh8ay/PmaWaGYfmdkbwXx76Ve+me00s+1mtjVoC/VrsTniGtxmlgj8AvgGMBy428yGx7OmJngRmH5F22PAOncfBqwL5iHSz2HBYx6wqJVqbIpK4O/cfTgwEXg4+LcJe9/KgKnufjMwEphuZhOBnwI/d/frgdPAA8H6DwCng/afB+u1ZT8E9kbNt5d+AXzN3UdGnfoX9tdi07l73B7AJODNqPnHgcfjWVMT+zEE2BU1vw/IDqaziZynDvAr4O661mvrD+B1YFp76hvQCfgQmEDkAo6koL3mdQm8CUwKppOC9SzetV+lPwOIBNhU4A3A2kO/ghrzgV5XtLWb12JjH/EeKukPHImaPxq0hV2WuxcG08eArGA6lP0N/oweBWymHfQtGE7YDhQBa4FPgTPuXhmsEl17Tb+C5SVAz9atuMH+D/A/gUvBfE/aR78AHHjLzLaZ2bygLfSvxaZqK1dOtlvu7mYW2lN3zKwzsAL4kbufNbOaZWHtm7tXASPNrDvwKnBDnEtqNjP7T0CRu28zsynxrqcFfMXdC8ysD7DWzD6JXhjW12JTxfuIuwAYGDU/IGgLu+Nmlg0Q/CwK2kPVXzNLJhLaf3D3V4LmdtE3AHc/A7xDZAihu5lVH8hE117Tr2B5N+BUK5faELcA/8XM8oHlRIZL/i/h7xcA7l4Q/Cwi8p/teNrRa7Gx4h3cHwDDgk++U4DZwMo41xQLK4H7g+n7iYwPV7ffF3zqPREoifpTr02xyKH1b4C97v5s1KJQ983MegdH2phZOpFx+71EAnxWsNqV/aru7yxgvQcDp22Juz/u7gPcfQiR99F6d/8OIe8XgJllmFmX6mngb4FdhPy12CzxHmQHZgD7iYwz/q9419OE+pcBhUAFkbG0B4iMFa4DDgBvA5nBukbkLJpPgZ3A2HjXf41+fYXIuOIOYHvwmBH2vgEjgI+Cfu0C/jFozwG2ALnAn4DUoD0tmM8NlufEuw8N6OMU4I320q+gDx8Hj93VORH212JzHrpyUkQkZOI9VCIiIo2k4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZP4/aOAjA9p4Wi8AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "F5paWqo7tWL2" + }, + "source": [ + "## Policy Gradient\n", + "Now, we can build a simple policy network. The network will return one of action in the action space." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "J8tdmeD-tZew" + }, + "source": [ + "class PolicyGradientNetwork(nn.Module):\n", + "\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.fc1 = nn.Linear(8, 16)\n", + " self.fc2 = nn.Linear(16, 16)\n", + " self.fc3 = nn.Linear(16, 4)\n", + "\n", + " def forward(self, state):\n", + " hid = torch.tanh(self.fc1(state))\n", + " hid = torch.tanh(self.fc2(hid))\n", + " return F.softmax(self.fc3(hid), dim=-1)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ynbqJrhIFTC3" + }, + "source": [ + "Then, we need to build a simple agent. The agent will acts according to the output of the policy network above. There are a few things can be done by agent:\n", + "- `learn()`：update the policy network from log probabilities and rewards.\n", + "- `sample()`：After receiving observation from the environment, utilize policy network to tell which action to take. The return values of this function includes action and log probabilities. " + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "zZo-IxJx286z" + }, + "source": [ + "from torch.optim.lr_scheduler import StepLR\n", + "class PolicyGradientAgent():\n", + " \n", + " def __init__(self, network):\n", + " self.network = network\n", + " self.optimizer = optim.SGD(self.network.parameters(), lr=0.001)\n", + " \n", + " def forward(self, state):\n", + " return self.network(state)\n", + " def learn(self, log_probs, rewards):\n", + " loss = (-log_probs * rewards).sum() # You don't need to revise this to pass simple baseline (but you can)\n", + "\n", + " self.optimizer.zero_grad()\n", + " loss.backward()\n", + " self.optimizer.step()\n", + " \n", + " def sample(self, state):\n", + " action_prob = self.network(torch.FloatTensor(state))\n", + " action_dist = Categorical(action_prob)\n", + " action = action_dist.sample()\n", + " log_prob = action_dist.log_prob(action)\n", + " return action.item(), log_prob" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ehPlnTKyRZf9" + }, + "source": [ + "Lastly, build a network and agent to start training." + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "GfJIvML-RYjL" + }, + "source": [ + "network = PolicyGradientNetwork()\n", + "agent = PolicyGradientAgent(network)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ouv23glgf5Qt" + }, + "source": [ + "## Trainin Agent\n", + "\n", + "Now let's start to train our agent.\n", + "Through taking all the interactions between agent and environment as training data, the policy network can learn from all these attempts," + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000, + "referenced_widgets": [ + "de3a153737af485ea436d7e8393d8248", + "6345f8926212465291c04587353161f1", + "3aa84c8c097d4858a0c38f18dec8b060", + "6647e68cf064416ca593b990bed81edf", + "a25de7edbbee47cc8094f43125efa39b", + "a0bb5061ca0946ab89a89f9abadcebc8", + "d99d8e6bcbe0445eb7eddbfe31277635", + "5af1c6579f0f4c2eb03df4063749569e" + ] + }, + "id": "vg5rxBBaf38_", + "outputId": "a1b06e39-99d6-4233-eda0-a3d58e77ffed" + }, + "source": [ + "agent.network.train() # Switch network into training mode \n", + "EPISODE_PER_BATCH = 5 # update the agent every 5 episode\n", + "NUM_BATCH = 400 # totally update the agent for 400 time\n", + "\n", + "avg_total_rewards, avg_final_rewards = [], []\n", + "\n", + "prg_bar = tqdm(range(NUM_BATCH))\n", + "for batch in prg_bar:\n", + "\n", + " log_probs, rewards = [], []\n", + " total_rewards, final_rewards = [], []\n", + "\n", + " # collect trajectory\n", + " for episode in range(EPISODE_PER_BATCH):\n", + " \n", + " state = env.reset()\n", + " total_reward, total_step = 0, 0\n", + " seq_rewards = []\n", + " while True:\n", + "\n", + " action, log_prob = agent.sample(state) # at, log(at|st)\n", + " next_state, reward, done, _ = env.step(action)\n", + "\n", + " log_probs.append(log_prob) # [log(a1|s1), log(a2|s2), ...., log(at|st)]\n", + " # seq_rewards.append(reward)\n", + " state = next_state\n", + " total_reward += reward\n", + " total_step += 1\n", + " rewards.append(reward) # change here\n", + " # ! IMPORTANT !\n", + " # Current reward implementation: immediate reward, given action_list : a1, a2, a3 ......\n", + " # rewards : r1, r2 ,r3 ......\n", + " # medium：change \"rewards\" to accumulative decaying reward, given action_list : a1, a2, a3, ......\n", + " # rewards : r1+0.99*r2+0.99^2*r3+......, r2+0.99*r3+0.99^2*r4+...... , r3+0.99*r4+0.99^2*r5+ ......\n", + " # boss : implement DQN\n", + " if done:\n", + " final_rewards.append(reward)\n", + " total_rewards.append(total_reward)\n", + " \n", + " break\n", + "\n", + " print(f\"rewards looks like \", np.shape(rewards)) \n", + " print(f\"log_probs looks like \", np.shape(log_probs)) \n", + " # record training process\n", + " avg_total_reward = sum(total_rewards) / len(total_rewards)\n", + " avg_final_reward = sum(final_rewards) / len(final_rewards)\n", + " avg_total_rewards.append(avg_total_reward)\n", + " avg_final_rewards.append(avg_final_reward)\n", + " prg_bar.set_description(f\"Total: {avg_total_reward: 4.1f}, Final: {avg_final_reward: 4.1f}\")\n", + "\n", + " # update agent\n", + " # rewards = np.concatenate(rewards, axis=0)\n", + " rewards = (rewards - np.mean(rewards)) / (np.std(rewards) + 1e-9) # normalize the reward \n", + " agent.learn(torch.stack(log_probs), torch.from_numpy(rewards))\n", + " print(\"logs prob looks like \", torch.stack(log_probs).size())\n", + " print(\"torch.from_numpy(rewards) looks like \", torch.from_numpy(rewards).size())" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "de3a153737af485ea436d7e8393d8248", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=400.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "rewards looks like (448,)\n", + "log_probs looks like (448,)\n", + "logs prob looks like torch.Size([448])\n", + "torch.from_numpy(rewards) looks like torch.Size([448])\n", + "rewards looks like (515,)\n", + "log_probs looks like (515,)\n", + "logs prob looks like torch.Size([515])\n", + "torch.from_numpy(rewards) looks like torch.Size([515])\n", + "rewards looks like (392,)\n", + "log_probs looks like (392,)\n", + "logs prob looks like torch.Size([392])\n", + "torch.from_numpy(rewards) looks like torch.Size([392])\n", + "rewards looks like (518,)\n", + "log_probs looks like (518,)\n", + "logs prob looks like torch.Size([518])\n", + "torch.from_numpy(rewards) looks like torch.Size([518])\n", + "rewards looks like (472,)\n", + "log_probs looks like (472,)\n", + "logs prob looks like torch.Size([472])\n", + "torch.from_numpy(rewards) looks like torch.Size([472])\n", + "rewards looks like (530,)\n", + "log_probs looks like (530,)\n", + "logs prob looks like torch.Size([530])\n", + "torch.from_numpy(rewards) looks like torch.Size([530])\n", + "rewards looks like (463,)\n", + "log_probs looks like (463,)\n", + "logs prob looks like torch.Size([463])\n", + "torch.from_numpy(rewards) looks like torch.Size([463])\n", + "rewards looks like (540,)\n", + "log_probs looks like (540,)\n", + "logs prob looks like torch.Size([540])\n", + "torch.from_numpy(rewards) looks like torch.Size([540])\n", + "rewards looks like (513,)\n", + "log_probs looks like (513,)\n", + "logs prob looks like torch.Size([513])\n", + "torch.from_numpy(rewards) looks like torch.Size([513])\n", + "rewards looks like (449,)\n", + "log_probs looks like (449,)\n", + "logs prob looks like torch.Size([449])\n", + "torch.from_numpy(rewards) looks like torch.Size([449])\n", + "rewards looks like (602,)\n", + "log_probs looks like (602,)\n", + "logs prob looks like torch.Size([602])\n", + "torch.from_numpy(rewards) looks like torch.Size([602])\n", + "rewards looks like (542,)\n", + "log_probs looks like (542,)\n", + "logs prob looks like torch.Size([542])\n", + "torch.from_numpy(rewards) looks like torch.Size([542])\n", + "rewards looks like (503,)\n", + "log_probs looks like (503,)\n", + "logs prob looks like torch.Size([503])\n", + "torch.from_numpy(rewards) looks like torch.Size([503])\n", + "rewards looks like (470,)\n", + "log_probs looks like (470,)\n", + "logs prob looks like torch.Size([470])\n", + "torch.from_numpy(rewards) looks like torch.Size([470])\n", + "rewards looks like (518,)\n", + "log_probs looks like (518,)\n", + "logs prob looks like torch.Size([518])\n", + "torch.from_numpy(rewards) looks like torch.Size([518])\n", + "rewards looks like (421,)\n", + "log_probs looks like (421,)\n", + "logs prob looks like torch.Size([421])\n", + "torch.from_numpy(rewards) looks like torch.Size([421])\n", + "rewards looks like (592,)\n", + "log_probs looks like (592,)\n", + "logs prob looks like torch.Size([592])\n", + "torch.from_numpy(rewards) looks like torch.Size([592])\n", + "rewards looks like (520,)\n", + "log_probs looks like (520,)\n", + "logs prob looks like torch.Size([520])\n", + "torch.from_numpy(rewards) looks like torch.Size([520])\n", + "rewards looks like (494,)\n", + "log_probs looks like (494,)\n", + "logs prob looks like torch.Size([494])\n", + "torch.from_numpy(rewards) looks like torch.Size([494])\n", + "rewards looks like (461,)\n", + "log_probs looks like (461,)\n", + "logs prob looks like torch.Size([461])\n", + "torch.from_numpy(rewards) looks like torch.Size([461])\n", + "rewards looks like (572,)\n", + "log_probs looks like (572,)\n", + "logs prob looks like torch.Size([572])\n", + "torch.from_numpy(rewards) looks like torch.Size([572])\n", + "rewards looks like (593,)\n", + "log_probs looks like (593,)\n", + "logs prob looks like torch.Size([593])\n", + "torch.from_numpy(rewards) looks like torch.Size([593])\n", + "rewards looks like (569,)\n", + "log_probs looks like (569,)\n", + "logs prob looks like torch.Size([569])\n", + "torch.from_numpy(rewards) looks like torch.Size([569])\n", + "rewards looks like (546,)\n", + "log_probs looks like (546,)\n", + "logs prob looks like torch.Size([546])\n", + "torch.from_numpy(rewards) looks like torch.Size([546])\n", + "rewards looks like (612,)\n", + "log_probs looks like (612,)\n", + "logs prob looks like torch.Size([612])\n", + "torch.from_numpy(rewards) looks like torch.Size([612])\n", + "rewards looks like (534,)\n", + "log_probs looks like (534,)\n", + "logs prob looks like torch.Size([534])\n", + "torch.from_numpy(rewards) looks like torch.Size([534])\n", + "rewards looks like (513,)\n", + "log_probs looks like (513,)\n", + "logs prob looks like torch.Size([513])\n", + "torch.from_numpy(rewards) looks like torch.Size([513])\n", + "rewards looks like (513,)\n", + "log_probs looks like (513,)\n", + "logs prob looks like torch.Size([513])\n", + "torch.from_numpy(rewards) looks like torch.Size([513])\n", + "rewards looks like (535,)\n", + "log_probs looks like (535,)\n", + "logs prob looks like torch.Size([535])\n", + "torch.from_numpy(rewards) looks like torch.Size([535])\n", + "rewards looks like (533,)\n", + "log_probs looks like (533,)\n", + "logs prob looks like torch.Size([533])\n", + "torch.from_numpy(rewards) looks like torch.Size([533])\n", + "rewards looks like (521,)\n", + "log_probs looks like (521,)\n", + "logs prob looks like torch.Size([521])\n", + "torch.from_numpy(rewards) looks like torch.Size([521])\n", + "rewards looks like (566,)\n", + "log_probs looks like (566,)\n", + "logs prob looks like torch.Size([566])\n", + "torch.from_numpy(rewards) looks like torch.Size([566])\n", + "rewards looks like (586,)\n", + "log_probs looks like (586,)\n", + "logs prob looks like torch.Size([586])\n", + "torch.from_numpy(rewards) looks like torch.Size([586])\n", + "rewards looks like (575,)\n", + "log_probs looks like (575,)\n", + "logs prob looks like torch.Size([575])\n", + "torch.from_numpy(rewards) looks like torch.Size([575])\n", + "rewards looks like (709,)\n", + "log_probs looks like (709,)\n", + "logs prob looks like torch.Size([709])\n", + "torch.from_numpy(rewards) looks like torch.Size([709])\n", + "rewards looks like (486,)\n", + "log_probs looks like (486,)\n", + "logs prob looks like torch.Size([486])\n", + "torch.from_numpy(rewards) looks like torch.Size([486])\n", + "rewards looks like (557,)\n", + "log_probs looks like (557,)\n", + "logs prob looks like torch.Size([557])\n", + "torch.from_numpy(rewards) looks like torch.Size([557])\n", + "rewards looks like (517,)\n", + "log_probs looks like (517,)\n", + "logs prob looks like torch.Size([517])\n", + "torch.from_numpy(rewards) looks like torch.Size([517])\n", + "rewards looks like (550,)\n", + "log_probs looks like (550,)\n", + "logs prob looks like torch.Size([550])\n", + "torch.from_numpy(rewards) looks like torch.Size([550])\n", + "rewards looks like (690,)\n", + "log_probs looks like (690,)\n", + "logs prob looks like torch.Size([690])\n", + "torch.from_numpy(rewards) looks like torch.Size([690])\n", + "rewards looks like (591,)\n", + "log_probs looks like (591,)\n", + "logs prob looks like torch.Size([591])\n", + "torch.from_numpy(rewards) looks like torch.Size([591])\n", + "rewards looks like (689,)\n", + "log_probs looks like (689,)\n", + "logs prob looks like torch.Size([689])\n", + "torch.from_numpy(rewards) looks like torch.Size([689])\n", + "rewards looks like (1059,)\n", + "log_probs looks like (1059,)\n", + "logs prob looks like torch.Size([1059])\n", + "torch.from_numpy(rewards) looks like torch.Size([1059])\n", + "rewards looks like (619,)\n", + "log_probs looks like (619,)\n", + "logs prob looks like torch.Size([619])\n", + "torch.from_numpy(rewards) looks like torch.Size([619])\n", + "rewards looks like (527,)\n", + "log_probs looks like (527,)\n", + "logs prob looks like torch.Size([527])\n", + "torch.from_numpy(rewards) looks like torch.Size([527])\n", + "rewards looks like (514,)\n", + "log_probs looks like (514,)\n", + "logs prob looks like torch.Size([514])\n", + "torch.from_numpy(rewards) looks like torch.Size([514])\n", + "rewards looks like (655,)\n", + "log_probs looks like (655,)\n", + "logs prob looks like torch.Size([655])\n", + "torch.from_numpy(rewards) looks like torch.Size([655])\n", + "rewards looks like (667,)\n", + "log_probs looks like (667,)\n", + "logs prob looks like torch.Size([667])\n", + "torch.from_numpy(rewards) looks like torch.Size([667])\n", + "rewards looks like (712,)\n", + "log_probs looks like (712,)\n", + "logs prob looks like torch.Size([712])\n", + "torch.from_numpy(rewards) looks like torch.Size([712])\n", + "rewards looks like (636,)\n", + "log_probs looks like (636,)\n", + "logs prob looks like torch.Size([636])\n", + "torch.from_numpy(rewards) looks like torch.Size([636])\n", + "rewards looks like (620,)\n", + "log_probs looks like (620,)\n", + "logs prob looks like torch.Size([620])\n", + "torch.from_numpy(rewards) looks like torch.Size([620])\n", + "rewards looks like (543,)\n", + "log_probs looks like (543,)\n", + "logs prob looks like torch.Size([543])\n", + "torch.from_numpy(rewards) looks like torch.Size([543])\n", + "rewards looks like (586,)\n", + "log_probs looks like (586,)\n", + "logs prob looks like torch.Size([586])\n", + "torch.from_numpy(rewards) looks like torch.Size([586])\n", + "rewards looks like (498,)\n", + "log_probs looks like (498,)\n", + "logs prob looks like torch.Size([498])\n", + "torch.from_numpy(rewards) looks like torch.Size([498])\n", + "rewards looks like (586,)\n", + "log_probs looks like (586,)\n", + "logs prob looks like torch.Size([586])\n", + "torch.from_numpy(rewards) looks like torch.Size([586])\n", + "rewards looks like (591,)\n", + "log_probs looks like (591,)\n", + "logs prob looks like torch.Size([591])\n", + "torch.from_numpy(rewards) looks like torch.Size([591])\n", + "rewards looks like (693,)\n", + "log_probs looks like (693,)\n", + "logs prob looks like torch.Size([693])\n", + "torch.from_numpy(rewards) looks like torch.Size([693])\n", + "rewards looks like (648,)\n", + "log_probs looks like (648,)\n", + "logs prob looks like torch.Size([648])\n", + "torch.from_numpy(rewards) looks like torch.Size([648])\n", + "rewards looks like (513,)\n", + "log_probs looks like (513,)\n", + "logs prob looks like torch.Size([513])\n", + "torch.from_numpy(rewards) looks like torch.Size([513])\n", + "rewards looks like (574,)\n", + "log_probs looks like (574,)\n", + "logs prob looks like torch.Size([574])\n", + "torch.from_numpy(rewards) looks like torch.Size([574])\n", + "rewards looks like (718,)\n", + "log_probs looks like (718,)\n", + "logs prob looks like torch.Size([718])\n", + "torch.from_numpy(rewards) looks like torch.Size([718])\n", + "rewards looks like (730,)\n", + "log_probs looks like (730,)\n", + "logs prob looks like torch.Size([730])\n", + "torch.from_numpy(rewards) looks like torch.Size([730])\n", + "rewards looks like (668,)\n", + "log_probs looks like (668,)\n", + "logs prob looks like torch.Size([668])\n", + "torch.from_numpy(rewards) looks like torch.Size([668])\n", + "rewards looks like (754,)\n", + "log_probs looks like (754,)\n", + "logs prob looks like torch.Size([754])\n", + "torch.from_numpy(rewards) looks like torch.Size([754])\n", + "rewards looks like (712,)\n", + "log_probs looks like (712,)\n", + "logs prob looks like torch.Size([712])\n", + "torch.from_numpy(rewards) looks like torch.Size([712])\n", + "rewards looks like (470,)\n", + "log_probs looks like (470,)\n", + "logs prob looks like torch.Size([470])\n", + "torch.from_numpy(rewards) looks like torch.Size([470])\n", + "rewards looks like (665,)\n", + "log_probs looks like (665,)\n", + "logs prob looks like torch.Size([665])\n", + "torch.from_numpy(rewards) looks like torch.Size([665])\n", + "rewards looks like (585,)\n", + "log_probs looks like (585,)\n", + "logs prob looks like torch.Size([585])\n", + "torch.from_numpy(rewards) looks like torch.Size([585])\n", + "rewards looks like (512,)\n", + "log_probs looks like (512,)\n", + "logs prob looks like torch.Size([512])\n", + "torch.from_numpy(rewards) looks like torch.Size([512])\n", + "rewards looks like (702,)\n", + "log_probs looks like (702,)\n", + "logs prob looks like torch.Size([702])\n", + "torch.from_numpy(rewards) looks like torch.Size([702])\n", + "rewards looks like (596,)\n", + "log_probs looks like (596,)\n", + "logs prob looks like torch.Size([596])\n", + "torch.from_numpy(rewards) looks like torch.Size([596])\n", + "rewards looks like (626,)\n", + "log_probs looks like (626,)\n", + "logs prob looks like torch.Size([626])\n", + "torch.from_numpy(rewards) looks like torch.Size([626])\n", + "rewards looks like (566,)\n", + "log_probs looks like (566,)\n", + "logs prob looks like torch.Size([566])\n", + "torch.from_numpy(rewards) looks like torch.Size([566])\n", + "rewards looks like (717,)\n", + "log_probs looks like (717,)\n", + "logs prob looks like torch.Size([717])\n", + "torch.from_numpy(rewards) looks like torch.Size([717])\n", + "rewards looks like (708,)\n", + "log_probs looks like (708,)\n", + "logs prob looks like torch.Size([708])\n", + "torch.from_numpy(rewards) looks like torch.Size([708])\n", + "rewards looks like (565,)\n", + "log_probs looks like (565,)\n", + "logs prob looks like torch.Size([565])\n", + "torch.from_numpy(rewards) looks like torch.Size([565])\n", + "rewards looks like (450,)\n", + "log_probs looks like (450,)\n", + "logs prob looks like torch.Size([450])\n", + "torch.from_numpy(rewards) looks like torch.Size([450])\n", + "rewards looks like (584,)\n", + "log_probs looks like (584,)\n", + "logs prob looks like torch.Size([584])\n", + "torch.from_numpy(rewards) looks like torch.Size([584])\n", + "rewards looks like (670,)\n", + "log_probs looks like (670,)\n", + "logs prob looks like torch.Size([670])\n", + "torch.from_numpy(rewards) looks like torch.Size([670])\n", + "rewards looks like (691,)\n", + "log_probs looks like (691,)\n", + "logs prob looks like torch.Size([691])\n", + "torch.from_numpy(rewards) looks like torch.Size([691])\n", + "rewards looks like (760,)\n", + "log_probs looks like (760,)\n", + "logs prob looks like torch.Size([760])\n", + "torch.from_numpy(rewards) looks like torch.Size([760])\n", + "rewards looks like (752,)\n", + "log_probs looks like (752,)\n", + "logs prob looks like torch.Size([752])\n", + "torch.from_numpy(rewards) looks like torch.Size([752])\n", + "rewards looks like (478,)\n", + "log_probs looks like (478,)\n", + "logs prob looks like torch.Size([478])\n", + "torch.from_numpy(rewards) looks like torch.Size([478])\n", + "rewards looks like (553,)\n", + "log_probs looks like (553,)\n", + "logs prob looks like torch.Size([553])\n", + "torch.from_numpy(rewards) looks like torch.Size([553])\n", + "rewards looks like (1660,)\n", + "log_probs looks like (1660,)\n", + "logs prob looks like torch.Size([1660])\n", + "torch.from_numpy(rewards) looks like torch.Size([1660])\n", + "rewards looks like (751,)\n", + "log_probs looks like (751,)\n", + "logs prob looks like torch.Size([751])\n", + "torch.from_numpy(rewards) looks like torch.Size([751])\n", + "rewards looks like (801,)\n", + "log_probs looks like (801,)\n", + "logs prob looks like torch.Size([801])\n", + "torch.from_numpy(rewards) looks like torch.Size([801])\n", + "rewards looks like (715,)\n", + "log_probs looks like (715,)\n", + "logs prob looks like torch.Size([715])\n", + "torch.from_numpy(rewards) looks like torch.Size([715])\n", + "rewards looks like (708,)\n", + "log_probs looks like (708,)\n", + "logs prob looks like torch.Size([708])\n", + "torch.from_numpy(rewards) looks like torch.Size([708])\n", + "rewards looks like (609,)\n", + "log_probs looks like (609,)\n", + "logs prob looks like torch.Size([609])\n", + "torch.from_numpy(rewards) looks like torch.Size([609])\n", + "rewards looks like (732,)\n", + "log_probs looks like (732,)\n", + "logs prob looks like torch.Size([732])\n", + "torch.from_numpy(rewards) looks like torch.Size([732])\n", + "rewards looks like (603,)\n", + "log_probs looks like (603,)\n", + "logs prob looks like torch.Size([603])\n", + "torch.from_numpy(rewards) looks like torch.Size([603])\n", + "rewards looks like (603,)\n", + "log_probs looks like (603,)\n", + "logs prob looks like torch.Size([603])\n", + "torch.from_numpy(rewards) looks like torch.Size([603])\n", + "rewards looks like (665,)\n", + "log_probs looks like (665,)\n", + "logs prob looks like torch.Size([665])\n", + "torch.from_numpy(rewards) looks like torch.Size([665])\n", + "rewards looks like (658,)\n", + "log_probs looks like (658,)\n", + "logs prob looks like torch.Size([658])\n", + "torch.from_numpy(rewards) looks like torch.Size([658])\n", + "rewards looks like (783,)\n", + "log_probs looks like (783,)\n", + "logs prob looks like torch.Size([783])\n", + "torch.from_numpy(rewards) looks like torch.Size([783])\n", + "rewards looks like (652,)\n", + "log_probs looks like (652,)\n", + "logs prob looks like torch.Size([652])\n", + "torch.from_numpy(rewards) looks like torch.Size([652])\n", + "rewards looks like (892,)\n", + "log_probs looks like (892,)\n", + "logs prob looks like torch.Size([892])\n", + "torch.from_numpy(rewards) looks like torch.Size([892])\n", + "rewards looks like (821,)\n", + "log_probs looks like (821,)\n", + "logs prob looks like torch.Size([821])\n", + "torch.from_numpy(rewards) looks like torch.Size([821])\n", + "rewards looks like (986,)\n", + "log_probs looks like (986,)\n", + "logs prob looks like torch.Size([986])\n", + "torch.from_numpy(rewards) looks like torch.Size([986])\n", + "rewards looks like (916,)\n", + "log_probs looks like (916,)\n", + "logs prob looks like torch.Size([916])\n", + "torch.from_numpy(rewards) looks like torch.Size([916])\n", + "rewards looks like (742,)\n", + "log_probs looks like (742,)\n", + "logs prob looks like torch.Size([742])\n", + "torch.from_numpy(rewards) looks like torch.Size([742])\n", + "rewards looks like (604,)\n", + "log_probs looks like (604,)\n", + "logs prob looks like torch.Size([604])\n", + "torch.from_numpy(rewards) looks like torch.Size([604])\n", + "rewards looks like (818,)\n", + "log_probs looks like (818,)\n", + "logs prob looks like torch.Size([818])\n", + "torch.from_numpy(rewards) looks like torch.Size([818])\n", + "rewards looks like (855,)\n", + "log_probs looks like (855,)\n", + "logs prob looks like torch.Size([855])\n", + "torch.from_numpy(rewards) looks like torch.Size([855])\n", + "rewards looks like (795,)\n", + "log_probs looks like (795,)\n", + "logs prob looks like torch.Size([795])\n", + "torch.from_numpy(rewards) looks like torch.Size([795])\n", + "rewards looks like (868,)\n", + "log_probs looks like (868,)\n", + "logs prob looks like torch.Size([868])\n", + "torch.from_numpy(rewards) looks like torch.Size([868])\n", + "rewards looks like (800,)\n", + "log_probs looks like (800,)\n", + "logs prob looks like torch.Size([800])\n", + "torch.from_numpy(rewards) looks like torch.Size([800])\n", + "rewards looks like (820,)\n", + "log_probs looks like (820,)\n", + "logs prob looks like torch.Size([820])\n", + "torch.from_numpy(rewards) looks like torch.Size([820])\n", + "rewards looks like (760,)\n", + "log_probs looks like (760,)\n", + "logs prob looks like torch.Size([760])\n", + "torch.from_numpy(rewards) looks like torch.Size([760])\n", + "rewards looks like (886,)\n", + "log_probs looks like (886,)\n", + "logs prob looks like torch.Size([886])\n", + "torch.from_numpy(rewards) looks like torch.Size([886])\n", + "rewards looks like (1027,)\n", + "log_probs looks like (1027,)\n", + "logs prob looks like torch.Size([1027])\n", + "torch.from_numpy(rewards) looks like torch.Size([1027])\n", + "rewards looks like (819,)\n", + "log_probs looks like (819,)\n", + "logs prob looks like torch.Size([819])\n", + "torch.from_numpy(rewards) looks like torch.Size([819])\n", + "rewards looks like (934,)\n", + "log_probs looks like (934,)\n", + "logs prob looks like torch.Size([934])\n", + "torch.from_numpy(rewards) looks like torch.Size([934])\n", + "rewards looks like (1648,)\n", + "log_probs looks like (1648,)\n", + "logs prob looks like torch.Size([1648])\n", + "torch.from_numpy(rewards) looks like torch.Size([1648])\n", + "rewards looks like (1057,)\n", + "log_probs looks like (1057,)\n", + "logs prob looks like torch.Size([1057])\n", + "torch.from_numpy(rewards) looks like torch.Size([1057])\n", + "rewards looks like (861,)\n", + "log_probs looks like (861,)\n", + "logs prob looks like torch.Size([861])\n", + "torch.from_numpy(rewards) looks like torch.Size([861])\n", + "rewards looks like (1533,)\n", + "log_probs looks like (1533,)\n", + "logs prob looks like torch.Size([1533])\n", + "torch.from_numpy(rewards) looks like torch.Size([1533])\n", + "rewards looks like (920,)\n", + "log_probs looks like (920,)\n", + "logs prob looks like torch.Size([920])\n", + "torch.from_numpy(rewards) looks like torch.Size([920])\n", + "rewards looks like (905,)\n", + "log_probs looks like (905,)\n", + "logs prob looks like torch.Size([905])\n", + "torch.from_numpy(rewards) looks like torch.Size([905])\n", + "rewards looks like (814,)\n", + "log_probs looks like (814,)\n", + "logs prob looks like torch.Size([814])\n", + "torch.from_numpy(rewards) looks like torch.Size([814])\n", + "rewards looks like (809,)\n", + "log_probs looks like (809,)\n", + "logs prob looks like torch.Size([809])\n", + "torch.from_numpy(rewards) looks like torch.Size([809])\n", + "rewards looks like (873,)\n", + "log_probs looks like (873,)\n", + "logs prob looks like torch.Size([873])\n", + "torch.from_numpy(rewards) looks like torch.Size([873])\n", + "rewards looks like (727,)\n", + "log_probs looks like (727,)\n", + "logs prob looks like torch.Size([727])\n", + "torch.from_numpy(rewards) looks like torch.Size([727])\n", + "rewards looks like (1129,)\n", + "log_probs looks like (1129,)\n", + "logs prob looks like torch.Size([1129])\n", + "torch.from_numpy(rewards) looks like torch.Size([1129])\n", + "rewards looks like (1394,)\n", + "log_probs looks like (1394,)\n", + "logs prob looks like torch.Size([1394])\n", + "torch.from_numpy(rewards) looks like torch.Size([1394])\n", + "rewards looks like (884,)\n", + "log_probs looks like (884,)\n", + "logs prob looks like torch.Size([884])\n", + "torch.from_numpy(rewards) looks like torch.Size([884])\n", + "rewards looks like (1132,)\n", + "log_probs looks like (1132,)\n", + "logs prob looks like torch.Size([1132])\n", + "torch.from_numpy(rewards) looks like torch.Size([1132])\n", + "rewards looks like (1007,)\n", + "log_probs looks like (1007,)\n", + "logs prob looks like torch.Size([1007])\n", + "torch.from_numpy(rewards) looks like torch.Size([1007])\n", + "rewards looks like (711,)\n", + "log_probs looks like (711,)\n", + "logs prob looks like torch.Size([711])\n", + "torch.from_numpy(rewards) looks like torch.Size([711])\n", + "rewards looks like (836,)\n", + "log_probs looks like (836,)\n", + "logs prob looks like torch.Size([836])\n", + "torch.from_numpy(rewards) looks like torch.Size([836])\n", + "rewards looks like (1514,)\n", + "log_probs looks like (1514,)\n", + "logs prob looks like torch.Size([1514])\n", + "torch.from_numpy(rewards) looks like torch.Size([1514])\n", + "rewards looks like (896,)\n", + "log_probs looks like (896,)\n", + "logs prob looks like torch.Size([896])\n", + "torch.from_numpy(rewards) looks like torch.Size([896])\n", + "rewards looks like (912,)\n", + "log_probs looks like (912,)\n", + "logs prob looks like torch.Size([912])\n", + "torch.from_numpy(rewards) looks like torch.Size([912])\n", + "rewards looks like (1478,)\n", + "log_probs looks like (1478,)\n", + "logs prob looks like torch.Size([1478])\n", + "torch.from_numpy(rewards) looks like torch.Size([1478])\n", + "rewards looks like (1279,)\n", + "log_probs looks like (1279,)\n", + "logs prob looks like torch.Size([1279])\n", + "torch.from_numpy(rewards) looks like torch.Size([1279])\n", + "rewards looks like (676,)\n", + "log_probs looks like (676,)\n", + "logs prob looks like torch.Size([676])\n", + "torch.from_numpy(rewards) looks like torch.Size([676])\n", + "rewards looks like (1768,)\n", + "log_probs looks like (1768,)\n", + "logs prob looks like torch.Size([1768])\n", + "torch.from_numpy(rewards) looks like torch.Size([1768])\n", + "rewards looks like (897,)\n", + "log_probs looks like (897,)\n", + "logs prob looks like torch.Size([897])\n", + "torch.from_numpy(rewards) looks like torch.Size([897])\n", + "rewards looks like (1119,)\n", + "log_probs looks like (1119,)\n", + "logs prob looks like torch.Size([1119])\n", + "torch.from_numpy(rewards) looks like torch.Size([1119])\n", + "rewards looks like (943,)\n", + "log_probs looks like (943,)\n", + "logs prob looks like torch.Size([943])\n", + "torch.from_numpy(rewards) looks like torch.Size([943])\n", + "rewards looks like (1255,)\n", + "log_probs looks like (1255,)\n", + "logs prob looks like torch.Size([1255])\n", + "torch.from_numpy(rewards) looks like torch.Size([1255])\n", + "rewards looks like (861,)\n", + "log_probs looks like (861,)\n", + "logs prob looks like torch.Size([861])\n", + "torch.from_numpy(rewards) looks like torch.Size([861])\n", + "rewards looks like (1149,)\n", + "log_probs looks like (1149,)\n", + "logs prob looks like torch.Size([1149])\n", + "torch.from_numpy(rewards) looks like torch.Size([1149])\n", + "rewards looks like (1229,)\n", + "log_probs looks like (1229,)\n", + "logs prob looks like torch.Size([1229])\n", + "torch.from_numpy(rewards) looks like torch.Size([1229])\n", + "rewards looks like (1680,)\n", + "log_probs looks like (1680,)\n", + "logs prob looks like torch.Size([1680])\n", + "torch.from_numpy(rewards) looks like torch.Size([1680])\n", + "rewards looks like (1731,)\n", + "log_probs looks like (1731,)\n", + "logs prob looks like torch.Size([1731])\n", + "torch.from_numpy(rewards) looks like torch.Size([1731])\n", + "rewards looks like (1017,)\n", + "log_probs looks like (1017,)\n", + "logs prob looks like torch.Size([1017])\n", + "torch.from_numpy(rewards) looks like torch.Size([1017])\n", + "rewards looks like (990,)\n", + "log_probs looks like (990,)\n", + "logs prob looks like torch.Size([990])\n", + "torch.from_numpy(rewards) looks like torch.Size([990])\n", + "rewards looks like (1020,)\n", + "log_probs looks like (1020,)\n", + "logs prob looks like torch.Size([1020])\n", + "torch.from_numpy(rewards) looks like torch.Size([1020])\n", + "rewards looks like (1240,)\n", + "log_probs looks like (1240,)\n", + "logs prob looks like torch.Size([1240])\n", + "torch.from_numpy(rewards) looks like torch.Size([1240])\n", + "rewards looks like (774,)\n", + "log_probs looks like (774,)\n", + "logs prob looks like torch.Size([774])\n", + "torch.from_numpy(rewards) looks like torch.Size([774])\n", + "rewards looks like (1069,)\n", + "log_probs looks like (1069,)\n", + "logs prob looks like torch.Size([1069])\n", + "torch.from_numpy(rewards) looks like torch.Size([1069])\n", + "rewards looks like (1355,)\n", + "log_probs looks like (1355,)\n", + "logs prob looks like torch.Size([1355])\n", + "torch.from_numpy(rewards) looks like torch.Size([1355])\n", + "rewards looks like (1556,)\n", + "log_probs looks like (1556,)\n", + "logs prob looks like torch.Size([1556])\n", + "torch.from_numpy(rewards) looks like torch.Size([1556])\n", + "rewards looks like (1840,)\n", + "log_probs looks like (1840,)\n", + "logs prob looks like torch.Size([1840])\n", + "torch.from_numpy(rewards) looks like torch.Size([1840])\n", + "rewards looks like (1352,)\n", + "log_probs looks like (1352,)\n", + "logs prob looks like torch.Size([1352])\n", + "torch.from_numpy(rewards) looks like torch.Size([1352])\n", + "rewards looks like (1617,)\n", + "log_probs looks like (1617,)\n", + "logs prob looks like torch.Size([1617])\n", + "torch.from_numpy(rewards) looks like torch.Size([1617])\n", + "rewards looks like (1637,)\n", + "log_probs looks like (1637,)\n", + "logs prob looks like torch.Size([1637])\n", + "torch.from_numpy(rewards) looks like torch.Size([1637])\n", + "rewards looks like (1606,)\n", + "log_probs looks like (1606,)\n", + "logs prob looks like torch.Size([1606])\n", + "torch.from_numpy(rewards) looks like torch.Size([1606])\n", + "rewards looks like (860,)\n", + "log_probs looks like (860,)\n", + "logs prob looks like torch.Size([860])\n", + "torch.from_numpy(rewards) looks like torch.Size([860])\n", + "rewards looks like (1780,)\n", + "log_probs looks like (1780,)\n", + "logs prob looks like torch.Size([1780])\n", + "torch.from_numpy(rewards) looks like torch.Size([1780])\n", + "rewards looks like (2248,)\n", + "log_probs looks like (2248,)\n", + "logs prob looks like torch.Size([2248])\n", + "torch.from_numpy(rewards) looks like torch.Size([2248])\n", + "rewards looks like (1410,)\n", + "log_probs looks like (1410,)\n", + "logs prob looks like torch.Size([1410])\n", + "torch.from_numpy(rewards) looks like torch.Size([1410])\n", + "rewards looks like (557,)\n", + "log_probs looks like (557,)\n", + "logs prob looks like torch.Size([557])\n", + "torch.from_numpy(rewards) looks like torch.Size([557])\n", + "rewards looks like (719,)\n", + "log_probs looks like (719,)\n", + "logs prob looks like torch.Size([719])\n", + "torch.from_numpy(rewards) looks like torch.Size([719])\n", + "rewards looks like (1919,)\n", + "log_probs looks like (1919,)\n", + "logs prob looks like torch.Size([1919])\n", + "torch.from_numpy(rewards) looks like torch.Size([1919])\n", + "rewards looks like (1250,)\n", + "log_probs looks like (1250,)\n", + "logs prob looks like torch.Size([1250])\n", + "torch.from_numpy(rewards) looks like torch.Size([1250])\n", + "rewards looks like (1054,)\n", + "log_probs looks like (1054,)\n", + "logs prob looks like torch.Size([1054])\n", + "torch.from_numpy(rewards) looks like torch.Size([1054])\n", + "rewards looks like (1276,)\n", + "log_probs looks like (1276,)\n", + "logs prob looks like torch.Size([1276])\n", + "torch.from_numpy(rewards) looks like torch.Size([1276])\n", + "rewards looks like (1040,)\n", + "log_probs looks like (1040,)\n", + "logs prob looks like torch.Size([1040])\n", + "torch.from_numpy(rewards) looks like torch.Size([1040])\n", + "rewards looks like (991,)\n", + "log_probs looks like (991,)\n", + "logs prob looks like torch.Size([991])\n", + "torch.from_numpy(rewards) looks like torch.Size([991])\n", + "rewards looks like (1390,)\n", + "log_probs looks like (1390,)\n", + "logs prob looks like torch.Size([1390])\n", + "torch.from_numpy(rewards) looks like torch.Size([1390])\n", + "rewards looks like (1349,)\n", + "log_probs looks like (1349,)\n", + "logs prob looks like torch.Size([1349])\n", + "torch.from_numpy(rewards) looks like torch.Size([1349])\n", + "rewards looks like (1332,)\n", + "log_probs looks like (1332,)\n", + "logs prob looks like torch.Size([1332])\n", + "torch.from_numpy(rewards) looks like torch.Size([1332])\n", + "rewards looks like (1378,)\n", + "log_probs looks like (1378,)\n", + "logs prob looks like torch.Size([1378])\n", + "torch.from_numpy(rewards) looks like torch.Size([1378])\n", + "rewards looks like (1967,)\n", + "log_probs looks like (1967,)\n", + "logs prob looks like torch.Size([1967])\n", + "torch.from_numpy(rewards) looks like torch.Size([1967])\n", + "rewards looks like (1789,)\n", + "log_probs looks like (1789,)\n", + "logs prob looks like torch.Size([1789])\n", + "torch.from_numpy(rewards) looks like torch.Size([1789])\n", + "rewards looks like (1325,)\n", + "log_probs looks like (1325,)\n", + "logs prob looks like torch.Size([1325])\n", + "torch.from_numpy(rewards) looks like torch.Size([1325])\n", + "rewards looks like (1685,)\n", + "log_probs looks like (1685,)\n", + "logs prob looks like torch.Size([1685])\n", + "torch.from_numpy(rewards) looks like torch.Size([1685])\n", + "rewards looks like (1895,)\n", + "log_probs looks like (1895,)\n", + "logs prob looks like torch.Size([1895])\n", + "torch.from_numpy(rewards) looks like torch.Size([1895])\n", + "rewards looks like (1920,)\n", + "log_probs looks like (1920,)\n", + "logs prob looks like torch.Size([1920])\n", + "torch.from_numpy(rewards) looks like torch.Size([1920])\n", + "rewards looks like (1522,)\n", + "log_probs looks like (1522,)\n", + "logs prob looks like torch.Size([1522])\n", + "torch.from_numpy(rewards) looks like torch.Size([1522])\n", + "rewards looks like (1173,)\n", + "log_probs looks like (1173,)\n", + "logs prob looks like torch.Size([1173])\n", + "torch.from_numpy(rewards) looks like torch.Size([1173])\n", + "rewards looks like (2136,)\n", + "log_probs looks like (2136,)\n", + "logs prob looks like torch.Size([2136])\n", + "torch.from_numpy(rewards) looks like torch.Size([2136])\n", + "rewards looks like (1696,)\n", + "log_probs looks like (1696,)\n", + "logs prob looks like torch.Size([1696])\n", + "torch.from_numpy(rewards) looks like torch.Size([1696])\n", + "rewards looks like (568,)\n", + "log_probs looks like (568,)\n", + "logs prob looks like torch.Size([568])\n", + "torch.from_numpy(rewards) looks like torch.Size([568])\n", + "rewards looks like (1475,)\n", + "log_probs looks like (1475,)\n", + "logs prob looks like torch.Size([1475])\n", + "torch.from_numpy(rewards) looks like torch.Size([1475])\n", + "rewards looks like (2470,)\n", + "log_probs looks like (2470,)\n", + "logs prob looks like torch.Size([2470])\n", + "torch.from_numpy(rewards) looks like torch.Size([2470])\n", + "rewards looks like (3053,)\n", + "log_probs looks like (3053,)\n", + "logs prob looks like torch.Size([3053])\n", + "torch.from_numpy(rewards) looks like torch.Size([3053])\n", + "rewards looks like (915,)\n", + "log_probs looks like (915,)\n", + "logs prob looks like torch.Size([915])\n", + "torch.from_numpy(rewards) looks like torch.Size([915])\n", + "rewards looks like (2049,)\n", + "log_probs looks like (2049,)\n", + "logs prob looks like torch.Size([2049])\n", + "torch.from_numpy(rewards) looks like torch.Size([2049])\n", + "rewards looks like (2068,)\n", + "log_probs looks like (2068,)\n", + "logs prob looks like torch.Size([2068])\n", + "torch.from_numpy(rewards) looks like torch.Size([2068])\n", + "rewards looks like (2528,)\n", + "log_probs looks like (2528,)\n", + "logs prob looks like torch.Size([2528])\n", + "torch.from_numpy(rewards) looks like torch.Size([2528])\n", + "rewards looks like (1839,)\n", + "log_probs looks like (1839,)\n", + "logs prob looks like torch.Size([1839])\n", + "torch.from_numpy(rewards) looks like torch.Size([1839])\n", + "rewards looks like (497,)\n", + "log_probs looks like (497,)\n", + "logs prob looks like torch.Size([497])\n", + "torch.from_numpy(rewards) looks like torch.Size([497])\n", + "rewards looks like (627,)\n", + "log_probs looks like (627,)\n", + "logs prob looks like torch.Size([627])\n", + "torch.from_numpy(rewards) looks like torch.Size([627])\n", + "rewards looks like (2354,)\n", + "log_probs looks like (2354,)\n", + "logs prob looks like torch.Size([2354])\n", + "torch.from_numpy(rewards) looks like torch.Size([2354])\n", + "rewards looks like (2394,)\n", + "log_probs looks like (2394,)\n", + "logs prob looks like torch.Size([2394])\n", + "torch.from_numpy(rewards) looks like torch.Size([2394])\n", + "rewards looks like (743,)\n", + "log_probs looks like (743,)\n", + "logs prob looks like torch.Size([743])\n", + "torch.from_numpy(rewards) looks like torch.Size([743])\n", + "rewards looks like (1572,)\n", + "log_probs looks like (1572,)\n", + "logs prob looks like torch.Size([1572])\n", + "torch.from_numpy(rewards) looks like torch.Size([1572])\n", + "rewards looks like (2575,)\n", + "log_probs looks like (2575,)\n", + "logs prob looks like torch.Size([2575])\n", + "torch.from_numpy(rewards) looks like torch.Size([2575])\n", + "rewards looks like (2226,)\n", + "log_probs looks like (2226,)\n", + "logs prob looks like torch.Size([2226])\n", + "torch.from_numpy(rewards) looks like torch.Size([2226])\n", + "rewards looks like (541,)\n", + "log_probs looks like (541,)\n", + "logs prob looks like torch.Size([541])\n", + "torch.from_numpy(rewards) looks like torch.Size([541])\n", + "rewards looks like (820,)\n", + "log_probs looks like (820,)\n", + "logs prob looks like torch.Size([820])\n", + "torch.from_numpy(rewards) looks like torch.Size([820])\n", + "rewards looks like (2584,)\n", + "log_probs looks like (2584,)\n", + "logs prob looks like torch.Size([2584])\n", + "torch.from_numpy(rewards) looks like torch.Size([2584])\n", + "rewards looks like (1792,)\n", + "log_probs looks like (1792,)\n", + "logs prob looks like torch.Size([1792])\n", + "torch.from_numpy(rewards) looks like torch.Size([1792])\n", + "rewards looks like (1613,)\n", + "log_probs looks like (1613,)\n", + "logs prob looks like torch.Size([1613])\n", + "torch.from_numpy(rewards) looks like torch.Size([1613])\n", + "rewards looks like (4300,)\n", + "log_probs looks like (4300,)\n", + "logs prob looks like torch.Size([4300])\n", + "torch.from_numpy(rewards) looks like torch.Size([4300])\n", + "rewards looks like (1602,)\n", + "log_probs looks like (1602,)\n", + "logs prob looks like torch.Size([1602])\n", + "torch.from_numpy(rewards) looks like torch.Size([1602])\n", + "rewards looks like (3313,)\n", + "log_probs looks like (3313,)\n", + "logs prob looks like torch.Size([3313])\n", + "torch.from_numpy(rewards) looks like torch.Size([3313])\n", + "rewards looks like (1538,)\n", + "log_probs looks like (1538,)\n", + "logs prob looks like torch.Size([1538])\n", + "torch.from_numpy(rewards) looks like torch.Size([1538])\n", + "rewards looks like (1824,)\n", + "log_probs looks like (1824,)\n", + "logs prob looks like torch.Size([1824])\n", + "torch.from_numpy(rewards) looks like torch.Size([1824])\n", + "rewards looks like (1320,)\n", + "log_probs looks like (1320,)\n", + "logs prob looks like torch.Size([1320])\n", + "torch.from_numpy(rewards) looks like torch.Size([1320])\n", + "rewards looks like (2077,)\n", + "log_probs looks like (2077,)\n", + "logs prob looks like torch.Size([2077])\n", + "torch.from_numpy(rewards) looks like torch.Size([2077])\n", + "rewards looks like (1995,)\n", + "log_probs looks like (1995,)\n", + "logs prob looks like torch.Size([1995])\n", + "torch.from_numpy(rewards) looks like torch.Size([1995])\n", + "rewards looks like (1089,)\n", + "log_probs looks like (1089,)\n", + "logs prob looks like torch.Size([1089])\n", + "torch.from_numpy(rewards) looks like torch.Size([1089])\n", + "rewards looks like (1135,)\n", + "log_probs looks like (1135,)\n", + "logs prob looks like torch.Size([1135])\n", + "torch.from_numpy(rewards) looks like torch.Size([1135])\n", + "rewards looks like (1617,)\n", + "log_probs looks like (1617,)\n", + "logs prob looks like torch.Size([1617])\n", + "torch.from_numpy(rewards) looks like torch.Size([1617])\n", + "rewards looks like (942,)\n", + "log_probs looks like (942,)\n", + "logs prob looks like torch.Size([942])\n", + "torch.from_numpy(rewards) looks like torch.Size([942])\n", + "rewards looks like (2006,)\n", + "log_probs looks like (2006,)\n", + "logs prob looks like torch.Size([2006])\n", + "torch.from_numpy(rewards) looks like torch.Size([2006])\n", + "rewards looks like (2204,)\n", + "log_probs looks like (2204,)\n", + "logs prob looks like torch.Size([2204])\n", + "torch.from_numpy(rewards) looks like torch.Size([2204])\n", + "rewards looks like (1060,)\n", + "log_probs looks like (1060,)\n", + "logs prob looks like torch.Size([1060])\n", + "torch.from_numpy(rewards) looks like torch.Size([1060])\n", + "rewards looks like (1994,)\n", + "log_probs looks like (1994,)\n", + "logs prob looks like torch.Size([1994])\n", + "torch.from_numpy(rewards) looks like torch.Size([1994])\n", + "rewards looks like (1118,)\n", + "log_probs looks like (1118,)\n", + "logs prob looks like torch.Size([1118])\n", + "torch.from_numpy(rewards) looks like torch.Size([1118])\n", + "rewards looks like (1298,)\n", + "log_probs looks like (1298,)\n", + "logs prob looks like torch.Size([1298])\n", + "torch.from_numpy(rewards) looks like torch.Size([1298])\n", + "rewards looks like (1377,)\n", + "log_probs looks like (1377,)\n", + "logs prob looks like torch.Size([1377])\n", + "torch.from_numpy(rewards) looks like torch.Size([1377])\n", + "rewards looks like (1902,)\n", + "log_probs looks like (1902,)\n", + "logs prob looks like torch.Size([1902])\n", + "torch.from_numpy(rewards) looks like torch.Size([1902])\n", + "rewards looks like (1982,)\n", + "log_probs looks like (1982,)\n", + "logs prob looks like torch.Size([1982])\n", + "torch.from_numpy(rewards) looks like torch.Size([1982])\n", + "rewards looks like (1625,)\n", + "log_probs looks like (1625,)\n", + "logs prob looks like torch.Size([1625])\n", + "torch.from_numpy(rewards) looks like torch.Size([1625])\n", + "rewards looks like (1947,)\n", + "log_probs looks like (1947,)\n", + "logs prob looks like torch.Size([1947])\n", + "torch.from_numpy(rewards) looks like torch.Size([1947])\n", + "rewards looks like (1589,)\n", + "log_probs looks like (1589,)\n", + "logs prob looks like torch.Size([1589])\n", + "torch.from_numpy(rewards) looks like torch.Size([1589])\n", + "rewards looks like (1625,)\n", + "log_probs looks like (1625,)\n", + "logs prob looks like torch.Size([1625])\n", + "torch.from_numpy(rewards) looks like torch.Size([1625])\n", + "rewards looks like (1492,)\n", + "log_probs looks like (1492,)\n", + "logs prob looks like torch.Size([1492])\n", + "torch.from_numpy(rewards) looks like torch.Size([1492])\n", + "rewards looks like (1347,)\n", + "log_probs looks like (1347,)\n", + "logs prob looks like torch.Size([1347])\n", + "torch.from_numpy(rewards) looks like torch.Size([1347])\n", + "rewards looks like (2110,)\n", + "log_probs looks like (2110,)\n", + "logs prob looks like torch.Size([2110])\n", + "torch.from_numpy(rewards) looks like torch.Size([2110])\n", + "rewards looks like (877,)\n", + "log_probs looks like (877,)\n", + "logs prob looks like torch.Size([877])\n", + "torch.from_numpy(rewards) looks like torch.Size([877])\n", + "rewards looks like (1078,)\n", + "log_probs looks like (1078,)\n", + "logs prob looks like torch.Size([1078])\n", + "torch.from_numpy(rewards) looks like torch.Size([1078])\n", + "rewards looks like (2001,)\n", + "log_probs looks like (2001,)\n", + "logs prob looks like torch.Size([2001])\n", + "torch.from_numpy(rewards) looks like torch.Size([2001])\n", + "rewards looks like (1452,)\n", + "log_probs looks like (1452,)\n", + "logs prob looks like torch.Size([1452])\n", + "torch.from_numpy(rewards) looks like torch.Size([1452])\n", + "rewards looks like (1169,)\n", + "log_probs looks like (1169,)\n", + "logs prob looks like torch.Size([1169])\n", + "torch.from_numpy(rewards) looks like torch.Size([1169])\n", + "rewards looks like (1977,)\n", + "log_probs looks like (1977,)\n", + "logs prob looks like torch.Size([1977])\n", + "torch.from_numpy(rewards) looks like torch.Size([1977])\n", + "rewards looks like (1263,)\n", + "log_probs looks like (1263,)\n", + "logs prob looks like torch.Size([1263])\n", + "torch.from_numpy(rewards) looks like torch.Size([1263])\n", + "rewards looks like (2219,)\n", + "log_probs looks like (2219,)\n", + "logs prob looks like torch.Size([2219])\n", + "torch.from_numpy(rewards) looks like torch.Size([2219])\n", + "rewards looks like (1732,)\n", + "log_probs looks like (1732,)\n", + "logs prob looks like torch.Size([1732])\n", + "torch.from_numpy(rewards) looks like torch.Size([1732])\n", + "rewards looks like (1413,)\n", + "log_probs looks like (1413,)\n", + "logs prob looks like torch.Size([1413])\n", + "torch.from_numpy(rewards) looks like torch.Size([1413])\n", + "rewards looks like (1099,)\n", + "log_probs looks like (1099,)\n", + "logs prob looks like torch.Size([1099])\n", + "torch.from_numpy(rewards) looks like torch.Size([1099])\n", + "rewards looks like (1184,)\n", + "log_probs looks like (1184,)\n", + "logs prob looks like torch.Size([1184])\n", + "torch.from_numpy(rewards) looks like torch.Size([1184])\n", + "rewards looks like (1148,)\n", + "log_probs looks like (1148,)\n", + "logs prob looks like torch.Size([1148])\n", + "torch.from_numpy(rewards) looks like torch.Size([1148])\n", + "rewards looks like (1339,)\n", + "log_probs looks like (1339,)\n", + "logs prob looks like torch.Size([1339])\n", + "torch.from_numpy(rewards) looks like torch.Size([1339])\n", + "rewards looks like (2095,)\n", + "log_probs looks like (2095,)\n", + "logs prob looks like torch.Size([2095])\n", + "torch.from_numpy(rewards) looks like torch.Size([2095])\n", + "rewards looks like (1514,)\n", + "log_probs looks like (1514,)\n", + "logs prob looks like torch.Size([1514])\n", + "torch.from_numpy(rewards) looks like torch.Size([1514])\n", + "rewards looks like (1276,)\n", + "log_probs looks like (1276,)\n", + "logs prob looks like torch.Size([1276])\n", + "torch.from_numpy(rewards) looks like torch.Size([1276])\n", + "rewards looks like (1277,)\n", + "log_probs looks like (1277,)\n", + "logs prob looks like torch.Size([1277])\n", + "torch.from_numpy(rewards) looks like torch.Size([1277])\n", + "rewards looks like (1453,)\n", + "log_probs looks like (1453,)\n", + "logs prob looks like torch.Size([1453])\n", + "torch.from_numpy(rewards) looks like torch.Size([1453])\n", + "rewards looks like (1467,)\n", + "log_probs looks like (1467,)\n", + "logs prob looks like torch.Size([1467])\n", + "torch.from_numpy(rewards) looks like torch.Size([1467])\n", + "rewards looks like (1383,)\n", + "log_probs looks like (1383,)\n", + "logs prob looks like torch.Size([1383])\n", + "torch.from_numpy(rewards) looks like torch.Size([1383])\n", + "rewards looks like (1741,)\n", + "log_probs looks like (1741,)\n", + "logs prob looks like torch.Size([1741])\n", + "torch.from_numpy(rewards) looks like torch.Size([1741])\n", + "rewards looks like (1039,)\n", + "log_probs looks like (1039,)\n", + "logs prob looks like torch.Size([1039])\n", + "torch.from_numpy(rewards) looks like torch.Size([1039])\n", + "rewards looks like (1063,)\n", + "log_probs looks like (1063,)\n", + "logs prob looks like torch.Size([1063])\n", + "torch.from_numpy(rewards) looks like torch.Size([1063])\n", + "rewards looks like (1731,)\n", + "log_probs looks like (1731,)\n", + "logs prob looks like torch.Size([1731])\n", + "torch.from_numpy(rewards) looks like torch.Size([1731])\n", + "rewards looks like (2661,)\n", + "log_probs looks like (2661,)\n", + "logs prob looks like torch.Size([2661])\n", + "torch.from_numpy(rewards) looks like torch.Size([2661])\n", + "rewards looks like (704,)\n", + "log_probs looks like (704,)\n", + "logs prob looks like torch.Size([704])\n", + "torch.from_numpy(rewards) looks like torch.Size([704])\n", + "rewards looks like (1389,)\n", + "log_probs looks like (1389,)\n", + "logs prob looks like torch.Size([1389])\n", + "torch.from_numpy(rewards) looks like torch.Size([1389])\n", + "rewards looks like (2131,)\n", + "log_probs looks like (2131,)\n", + "logs prob looks like torch.Size([2131])\n", + "torch.from_numpy(rewards) looks like torch.Size([2131])\n", + "rewards looks like (1779,)\n", + "log_probs looks like (1779,)\n", + "logs prob looks like torch.Size([1779])\n", + "torch.from_numpy(rewards) looks like torch.Size([1779])\n", + "rewards looks like (1415,)\n", + "log_probs looks like (1415,)\n", + "logs prob looks like torch.Size([1415])\n", + "torch.from_numpy(rewards) looks like torch.Size([1415])\n", + "rewards looks like (2320,)\n", + "log_probs looks like (2320,)\n", + "logs prob looks like torch.Size([2320])\n", + "torch.from_numpy(rewards) looks like torch.Size([2320])\n", + "rewards looks like (1147,)\n", + "log_probs looks like (1147,)\n", + "logs prob looks like torch.Size([1147])\n", + "torch.from_numpy(rewards) looks like torch.Size([1147])\n", + "rewards looks like (1022,)\n", + "log_probs looks like (1022,)\n", + "logs prob looks like torch.Size([1022])\n", + "torch.from_numpy(rewards) looks like torch.Size([1022])\n", + "rewards looks like (2141,)\n", + "log_probs looks like (2141,)\n", + "logs prob looks like torch.Size([2141])\n", + "torch.from_numpy(rewards) looks like torch.Size([2141])\n", + "rewards looks like (1362,)\n", + "log_probs looks like (1362,)\n", + "logs prob looks like torch.Size([1362])\n", + "torch.from_numpy(rewards) looks like torch.Size([1362])\n", + "rewards looks like (1450,)\n", + "log_probs looks like (1450,)\n", + "logs prob looks like torch.Size([1450])\n", + "torch.from_numpy(rewards) looks like torch.Size([1450])\n", + "rewards looks like (1546,)\n", + "log_probs looks like (1546,)\n", + "logs prob looks like torch.Size([1546])\n", + "torch.from_numpy(rewards) looks like torch.Size([1546])\n", + "rewards looks like (1166,)\n", + "log_probs looks like (1166,)\n", + "logs prob looks like torch.Size([1166])\n", + "torch.from_numpy(rewards) looks like torch.Size([1166])\n", + "rewards looks like (1647,)\n", + "log_probs looks like (1647,)\n", + "logs prob looks like torch.Size([1647])\n", + "torch.from_numpy(rewards) looks like torch.Size([1647])\n", + "rewards looks like (1205,)\n", + "log_probs looks like (1205,)\n", + "logs prob looks like torch.Size([1205])\n", + "torch.from_numpy(rewards) looks like torch.Size([1205])\n", + "rewards looks like (2098,)\n", + "log_probs looks like (2098,)\n", + "logs prob looks like torch.Size([2098])\n", + "torch.from_numpy(rewards) looks like torch.Size([2098])\n", + "rewards looks like (1940,)\n", + "log_probs looks like (1940,)\n", + "logs prob looks like torch.Size([1940])\n", + "torch.from_numpy(rewards) looks like torch.Size([1940])\n", + "rewards looks like (2191,)\n", + "log_probs looks like (2191,)\n", + "logs prob looks like torch.Size([2191])\n", + "torch.from_numpy(rewards) looks like torch.Size([2191])\n", + "rewards looks like (2740,)\n", + "log_probs looks like (2740,)\n", + "logs prob looks like torch.Size([2740])\n", + "torch.from_numpy(rewards) looks like torch.Size([2740])\n", + "rewards looks like (587,)\n", + "log_probs looks like (587,)\n", + "logs prob looks like torch.Size([587])\n", + "torch.from_numpy(rewards) looks like torch.Size([587])\n", + "rewards looks like (1063,)\n", + "log_probs looks like (1063,)\n", + "logs prob looks like torch.Size([1063])\n", + "torch.from_numpy(rewards) looks like torch.Size([1063])\n", + "rewards looks like (861,)\n", + "log_probs looks like (861,)\n", + "logs prob looks like torch.Size([861])\n", + "torch.from_numpy(rewards) looks like torch.Size([861])\n", + "rewards looks like (1051,)\n", + "log_probs looks like (1051,)\n", + "logs prob looks like torch.Size([1051])\n", + "torch.from_numpy(rewards) looks like torch.Size([1051])\n", + "rewards looks like (1389,)\n", + "log_probs looks like (1389,)\n", + "logs prob looks like torch.Size([1389])\n", + "torch.from_numpy(rewards) looks like torch.Size([1389])\n", + "rewards looks like (1152,)\n", + "log_probs looks like (1152,)\n", + "logs prob looks like torch.Size([1152])\n", + "torch.from_numpy(rewards) looks like torch.Size([1152])\n", + "rewards looks like (1103,)\n", + "log_probs looks like (1103,)\n", + "logs prob looks like torch.Size([1103])\n", + "torch.from_numpy(rewards) looks like torch.Size([1103])\n", + "rewards looks like (1887,)\n", + "log_probs looks like (1887,)\n", + "logs prob looks like torch.Size([1887])\n", + "torch.from_numpy(rewards) looks like torch.Size([1887])\n", + "rewards looks like (1753,)\n", + "log_probs looks like (1753,)\n", + "logs prob looks like torch.Size([1753])\n", + "torch.from_numpy(rewards) looks like torch.Size([1753])\n", + "rewards looks like (1372,)\n", + "log_probs looks like (1372,)\n", + "logs prob looks like torch.Size([1372])\n", + "torch.from_numpy(rewards) looks like torch.Size([1372])\n", + "rewards looks like (1056,)\n", + "log_probs looks like (1056,)\n", + "logs prob looks like torch.Size([1056])\n", + "torch.from_numpy(rewards) looks like torch.Size([1056])\n", + "rewards looks like (1465,)\n", + "log_probs looks like (1465,)\n", + "logs prob looks like torch.Size([1465])\n", + "torch.from_numpy(rewards) looks like torch.Size([1465])\n", + "rewards looks like (3297,)\n", + "log_probs looks like (3297,)\n", + "logs prob looks like torch.Size([3297])\n", + "torch.from_numpy(rewards) looks like torch.Size([3297])\n", + "rewards looks like (2492,)\n", + "log_probs looks like (2492,)\n", + "logs prob looks like torch.Size([2492])\n", + "torch.from_numpy(rewards) looks like torch.Size([2492])\n", + "rewards looks like (1580,)\n", + "log_probs looks like (1580,)\n", + "logs prob looks like torch.Size([1580])\n", + "torch.from_numpy(rewards) looks like torch.Size([1580])\n", + "rewards looks like (1357,)\n", + "log_probs looks like (1357,)\n", + "logs prob looks like torch.Size([1357])\n", + "torch.from_numpy(rewards) looks like torch.Size([1357])\n", + "rewards looks like (1227,)\n", + "log_probs looks like (1227,)\n", + "logs prob looks like torch.Size([1227])\n", + "torch.from_numpy(rewards) looks like torch.Size([1227])\n", + "rewards looks like (2123,)\n", + "log_probs looks like (2123,)\n", + "logs prob looks like torch.Size([2123])\n", + "torch.from_numpy(rewards) looks like torch.Size([2123])\n", + "rewards looks like (1864,)\n", + "log_probs looks like (1864,)\n", + "logs prob looks like torch.Size([1864])\n", + "torch.from_numpy(rewards) looks like torch.Size([1864])\n", + "rewards looks like (1324,)\n", + "log_probs looks like (1324,)\n", + "logs prob looks like torch.Size([1324])\n", + "torch.from_numpy(rewards) looks like torch.Size([1324])\n", + "rewards looks like (1281,)\n", + "log_probs looks like (1281,)\n", + "logs prob looks like torch.Size([1281])\n", + "torch.from_numpy(rewards) looks like torch.Size([1281])\n", + "rewards looks like (1366,)\n", + "log_probs looks like (1366,)\n", + "logs prob looks like torch.Size([1366])\n", + "torch.from_numpy(rewards) looks like torch.Size([1366])\n", + "rewards looks like (957,)\n", + "log_probs looks like (957,)\n", + "logs prob looks like torch.Size([957])\n", + "torch.from_numpy(rewards) looks like torch.Size([957])\n", + "rewards looks like (1187,)\n", + "log_probs looks like (1187,)\n", + "logs prob looks like torch.Size([1187])\n", + "torch.from_numpy(rewards) looks like torch.Size([1187])\n", + "rewards looks like (1625,)\n", + "log_probs looks like (1625,)\n", + "logs prob looks like torch.Size([1625])\n", + "torch.from_numpy(rewards) looks like torch.Size([1625])\n", + "rewards looks like (1605,)\n", + "log_probs looks like (1605,)\n", + "logs prob looks like torch.Size([1605])\n", + "torch.from_numpy(rewards) looks like torch.Size([1605])\n", + "rewards looks like (1015,)\n", + "log_probs looks like (1015,)\n", + "logs prob looks like torch.Size([1015])\n", + "torch.from_numpy(rewards) looks like torch.Size([1015])\n", + "rewards looks like (1565,)\n", + "log_probs looks like (1565,)\n", + "logs prob looks like torch.Size([1565])\n", + "torch.from_numpy(rewards) looks like torch.Size([1565])\n", + "rewards looks like (1353,)\n", + "log_probs looks like (1353,)\n", + "logs prob looks like torch.Size([1353])\n", + "torch.from_numpy(rewards) looks like torch.Size([1353])\n", + "rewards looks like (1321,)\n", + "log_probs looks like (1321,)\n", + "logs prob looks like torch.Size([1321])\n", + "torch.from_numpy(rewards) looks like torch.Size([1321])\n", + "rewards looks like (1074,)\n", + "log_probs looks like (1074,)\n", + "logs prob looks like torch.Size([1074])\n", + "torch.from_numpy(rewards) looks like torch.Size([1074])\n", + "rewards looks like (1301,)\n", + "log_probs looks like (1301,)\n", + "logs prob looks like torch.Size([1301])\n", + "torch.from_numpy(rewards) looks like torch.Size([1301])\n", + "rewards looks like (2105,)\n", + "log_probs looks like (2105,)\n", + "logs prob looks like torch.Size([2105])\n", + "torch.from_numpy(rewards) looks like torch.Size([2105])\n", + "rewards looks like (2008,)\n", + "log_probs looks like (2008,)\n", + "logs prob looks like torch.Size([2008])\n", + "torch.from_numpy(rewards) looks like torch.Size([2008])\n", + "rewards looks like (1885,)\n", + "log_probs looks like (1885,)\n", + "logs prob looks like torch.Size([1885])\n", + "torch.from_numpy(rewards) looks like torch.Size([1885])\n", + "rewards looks like (1184,)\n", + "log_probs looks like (1184,)\n", + "logs prob looks like torch.Size([1184])\n", + "torch.from_numpy(rewards) looks like torch.Size([1184])\n", + "rewards looks like (2551,)\n", + "log_probs looks like (2551,)\n", + "logs prob looks like torch.Size([2551])\n", + "torch.from_numpy(rewards) looks like torch.Size([2551])\n", + "rewards looks like (1330,)\n", + "log_probs looks like (1330,)\n", + "logs prob looks like torch.Size([1330])\n", + "torch.from_numpy(rewards) looks like torch.Size([1330])\n", + "rewards looks like (1510,)\n", + "log_probs looks like (1510,)\n", + "logs prob looks like torch.Size([1510])\n", + "torch.from_numpy(rewards) looks like torch.Size([1510])\n", + "rewards looks like (1330,)\n", + "log_probs looks like (1330,)\n", + "logs prob looks like torch.Size([1330])\n", + "torch.from_numpy(rewards) looks like torch.Size([1330])\n", + "rewards looks like (2157,)\n", + "log_probs looks like (2157,)\n", + "logs prob looks like torch.Size([2157])\n", + "torch.from_numpy(rewards) looks like torch.Size([2157])\n", + "rewards looks like (1276,)\n", + "log_probs looks like (1276,)\n", + "logs prob looks like torch.Size([1276])\n", + "torch.from_numpy(rewards) looks like torch.Size([1276])\n", + "rewards looks like (1188,)\n", + "log_probs looks like (1188,)\n", + "logs prob looks like torch.Size([1188])\n", + "torch.from_numpy(rewards) looks like torch.Size([1188])\n", + "rewards looks like (2381,)\n", + "log_probs looks like (2381,)\n", + "logs prob looks like torch.Size([2381])\n", + "torch.from_numpy(rewards) looks like torch.Size([2381])\n", + "rewards looks like (1450,)\n", + "log_probs looks like (1450,)\n", + "logs prob looks like torch.Size([1450])\n", + "torch.from_numpy(rewards) looks like torch.Size([1450])\n", + "rewards looks like (1612,)\n", + "log_probs looks like (1612,)\n", + "logs prob looks like torch.Size([1612])\n", + "torch.from_numpy(rewards) looks like torch.Size([1612])\n", + "rewards looks like (1780,)\n", + "log_probs looks like (1780,)\n", + "logs prob looks like torch.Size([1780])\n", + "torch.from_numpy(rewards) looks like torch.Size([1780])\n", + "rewards looks like (1350,)\n", + "log_probs looks like (1350,)\n", + "logs prob looks like torch.Size([1350])\n", + "torch.from_numpy(rewards) looks like torch.Size([1350])\n", + "rewards looks like (1459,)\n", + "log_probs looks like (1459,)\n", + "logs prob looks like torch.Size([1459])\n", + "torch.from_numpy(rewards) looks like torch.Size([1459])\n", + "rewards looks like (1958,)\n", + "log_probs looks like (1958,)\n", + "logs prob looks like torch.Size([1958])\n", + "torch.from_numpy(rewards) looks like torch.Size([1958])\n", + "rewards looks like (1325,)\n", + "log_probs looks like (1325,)\n", + "logs prob looks like torch.Size([1325])\n", + "torch.from_numpy(rewards) looks like torch.Size([1325])\n", + "rewards looks like (2168,)\n", + "log_probs looks like (2168,)\n", + "logs prob looks like torch.Size([2168])\n", + "torch.from_numpy(rewards) looks like torch.Size([2168])\n", + "rewards looks like (1682,)\n", + "log_probs looks like (1682,)\n", + "logs prob looks like torch.Size([1682])\n", + "torch.from_numpy(rewards) looks like torch.Size([1682])\n", + "rewards looks like (852,)\n", + "log_probs looks like (852,)\n", + "logs prob looks like torch.Size([852])\n", + "torch.from_numpy(rewards) looks like torch.Size([852])\n", + "rewards looks like (1757,)\n", + "log_probs looks like (1757,)\n", + "logs prob looks like torch.Size([1757])\n", + "torch.from_numpy(rewards) looks like torch.Size([1757])\n", + "rewards looks like (2313,)\n", + "log_probs looks like (2313,)\n", + "logs prob looks like torch.Size([2313])\n", + "torch.from_numpy(rewards) looks like torch.Size([2313])\n", + "rewards looks like (1662,)\n", + "log_probs looks like (1662,)\n", + "logs prob looks like torch.Size([1662])\n", + "torch.from_numpy(rewards) looks like torch.Size([1662])\n", + "rewards looks like (1559,)\n", + "log_probs looks like (1559,)\n", + "logs prob looks like torch.Size([1559])\n", + "torch.from_numpy(rewards) looks like torch.Size([1559])\n", + "rewards looks like (2077,)\n", + "log_probs looks like (2077,)\n", + "logs prob looks like torch.Size([2077])\n", + "torch.from_numpy(rewards) looks like torch.Size([2077])\n", + "rewards looks like (2119,)\n", + "log_probs looks like (2119,)\n", + "logs prob looks like torch.Size([2119])\n", + "torch.from_numpy(rewards) looks like torch.Size([2119])\n", + "rewards looks like (954,)\n", + "log_probs looks like (954,)\n", + "logs prob looks like torch.Size([954])\n", + "torch.from_numpy(rewards) looks like torch.Size([954])\n", + "rewards looks like (1797,)\n", + "log_probs looks like (1797,)\n", + "logs prob looks like torch.Size([1797])\n", + "torch.from_numpy(rewards) looks like torch.Size([1797])\n", + "rewards looks like (1579,)\n", + "log_probs looks like (1579,)\n", + "logs prob looks like torch.Size([1579])\n", + "torch.from_numpy(rewards) looks like torch.Size([1579])\n", + "rewards looks like (1277,)\n", + "log_probs looks like (1277,)\n", + "logs prob looks like torch.Size([1277])\n", + "torch.from_numpy(rewards) looks like torch.Size([1277])\n", + "rewards looks like (1196,)\n", + "log_probs looks like (1196,)\n", + "logs prob looks like torch.Size([1196])\n", + "torch.from_numpy(rewards) looks like torch.Size([1196])\n", + "rewards looks like (1294,)\n", + "log_probs looks like (1294,)\n", + "logs prob looks like torch.Size([1294])\n", + "torch.from_numpy(rewards) looks like torch.Size([1294])\n", + "rewards looks like (1318,)\n", + "log_probs looks like (1318,)\n", + "logs prob looks like torch.Size([1318])\n", + "torch.from_numpy(rewards) looks like torch.Size([1318])\n", + "rewards looks like (2605,)\n", + "log_probs looks like (2605,)\n", + "logs prob looks like torch.Size([2605])\n", + "torch.from_numpy(rewards) looks like torch.Size([2605])\n", + "rewards looks like (2002,)\n", + "log_probs looks like (2002,)\n", + "logs prob looks like torch.Size([2002])\n", + "torch.from_numpy(rewards) looks like torch.Size([2002])\n", + "rewards looks like (1354,)\n", + "log_probs looks like (1354,)\n", + "logs prob looks like torch.Size([1354])\n", + "torch.from_numpy(rewards) looks like torch.Size([1354])\n", + "rewards looks like (1785,)\n", + "log_probs looks like (1785,)\n", + "logs prob looks like torch.Size([1785])\n", + "torch.from_numpy(rewards) looks like torch.Size([1785])\n", + "rewards looks like (781,)\n", + "log_probs looks like (781,)\n", + "logs prob looks like torch.Size([781])\n", + "torch.from_numpy(rewards) looks like torch.Size([781])\n", + "rewards looks like (1965,)\n", + "log_probs looks like (1965,)\n", + "logs prob looks like torch.Size([1965])\n", + "torch.from_numpy(rewards) looks like torch.Size([1965])\n", + "rewards looks like (1135,)\n", + "log_probs looks like (1135,)\n", + "logs prob looks like torch.Size([1135])\n", + "torch.from_numpy(rewards) looks like torch.Size([1135])\n", + "rewards looks like (1672,)\n", + "log_probs looks like (1672,)\n", + "logs prob looks like torch.Size([1672])\n", + "torch.from_numpy(rewards) looks like torch.Size([1672])\n", + "rewards looks like (1278,)\n", + "log_probs looks like (1278,)\n", + "logs prob looks like torch.Size([1278])\n", + "torch.from_numpy(rewards) looks like torch.Size([1278])\n", + "rewards looks like (2499,)\n", + "log_probs looks like (2499,)\n", + "logs prob looks like torch.Size([2499])\n", + "torch.from_numpy(rewards) looks like torch.Size([2499])\n", + "rewards looks like (1275,)\n", + "log_probs looks like (1275,)\n", + "logs prob looks like torch.Size([1275])\n", + "torch.from_numpy(rewards) looks like torch.Size([1275])\n", + "rewards looks like (1144,)\n", + "log_probs looks like (1144,)\n", + "logs prob looks like torch.Size([1144])\n", + "torch.from_numpy(rewards) looks like torch.Size([1144])\n", + "rewards looks like (1605,)\n", + "log_probs looks like (1605,)\n", + "logs prob looks like torch.Size([1605])\n", + "torch.from_numpy(rewards) looks like torch.Size([1605])\n", + "rewards looks like (1178,)\n", + "log_probs looks like (1178,)\n", + "logs prob looks like torch.Size([1178])\n", + "torch.from_numpy(rewards) looks like torch.Size([1178])\n", + "rewards looks like (3269,)\n", + "log_probs looks like (3269,)\n", + "logs prob looks like torch.Size([3269])\n", + "torch.from_numpy(rewards) looks like torch.Size([3269])\n", + "rewards looks like (1492,)\n", + "log_probs looks like (1492,)\n", + "logs prob looks like torch.Size([1492])\n", + "torch.from_numpy(rewards) looks like torch.Size([1492])\n", + "rewards looks like (1285,)\n", + "log_probs looks like (1285,)\n", + "logs prob looks like torch.Size([1285])\n", + "torch.from_numpy(rewards) looks like torch.Size([1285])\n", + "rewards looks like (1687,)\n", + "log_probs looks like (1687,)\n", + "logs prob looks like torch.Size([1687])\n", + "torch.from_numpy(rewards) looks like torch.Size([1687])\n", + "rewards looks like (1124,)\n", + "log_probs looks like (1124,)\n", + "logs prob looks like torch.Size([1124])\n", + "torch.from_numpy(rewards) looks like torch.Size([1124])\n", + "rewards looks like (2043,)\n", + "log_probs looks like (2043,)\n", + "logs prob looks like torch.Size([2043])\n", + "torch.from_numpy(rewards) looks like torch.Size([2043])\n", + "rewards looks like (1280,)\n", + "log_probs looks like (1280,)\n", + "logs prob looks like torch.Size([1280])\n", + "torch.from_numpy(rewards) looks like torch.Size([1280])\n", + "rewards looks like (1418,)\n", + "log_probs looks like (1418,)\n", + "logs prob looks like torch.Size([1418])\n", + "torch.from_numpy(rewards) looks like torch.Size([1418])\n", + "rewards looks like (1365,)\n", + "log_probs looks like (1365,)\n", + "logs prob looks like torch.Size([1365])\n", + "torch.from_numpy(rewards) looks like torch.Size([1365])\n", + "rewards looks like (1091,)\n", + "log_probs looks like (1091,)\n", + "logs prob looks like torch.Size([1091])\n", + "torch.from_numpy(rewards) looks like torch.Size([1091])\n", + "rewards looks like (1279,)\n", + "log_probs looks like (1279,)\n", + "logs prob looks like torch.Size([1279])\n", + "torch.from_numpy(rewards) looks like torch.Size([1279])\n", + "rewards looks like (1109,)\n", + "log_probs looks like (1109,)\n", + "logs prob looks like torch.Size([1109])\n", + "torch.from_numpy(rewards) looks like torch.Size([1109])\n", + "rewards looks like (1285,)\n", + "log_probs looks like (1285,)\n", + "logs prob looks like torch.Size([1285])\n", + "torch.from_numpy(rewards) looks like torch.Size([1285])\n", + "rewards looks like (1222,)\n", + "log_probs looks like (1222,)\n", + "logs prob looks like torch.Size([1222])\n", + "torch.from_numpy(rewards) looks like torch.Size([1222])\n", + "rewards looks like (1538,)\n", + "log_probs looks like (1538,)\n", + "logs prob looks like torch.Size([1538])\n", + "torch.from_numpy(rewards) looks like torch.Size([1538])\n", + "rewards looks like (1139,)\n", + "log_probs looks like (1139,)\n", + "logs prob looks like torch.Size([1139])\n", + "torch.from_numpy(rewards) looks like torch.Size([1139])\n", + "rewards looks like (1354,)\n", + "log_probs looks like (1354,)\n", + "logs prob looks like torch.Size([1354])\n", + "torch.from_numpy(rewards) looks like torch.Size([1354])\n", + "rewards looks like (1166,)\n", + "log_probs looks like (1166,)\n", + "logs prob looks like torch.Size([1166])\n", + "torch.from_numpy(rewards) looks like torch.Size([1166])\n", + "rewards looks like (1348,)\n", + "log_probs looks like (1348,)\n", + "logs prob looks like torch.Size([1348])\n", + "torch.from_numpy(rewards) looks like torch.Size([1348])\n", + "rewards looks like (1347,)\n", + "log_probs looks like (1347,)\n", + "logs prob looks like torch.Size([1347])\n", + "torch.from_numpy(rewards) looks like torch.Size([1347])\n", + "rewards looks like (2059,)\n", + "log_probs looks like (2059,)\n", + "logs prob looks like torch.Size([2059])\n", + "torch.from_numpy(rewards) looks like torch.Size([2059])\n", + "rewards looks like (2021,)\n", + "log_probs looks like (2021,)\n", + "logs prob looks like torch.Size([2021])\n", + "torch.from_numpy(rewards) looks like torch.Size([2021])\n", + "rewards looks like (2232,)\n", + "log_probs looks like (2232,)\n", + "logs prob looks like torch.Size([2232])\n", + "torch.from_numpy(rewards) looks like torch.Size([2232])\n", + "rewards looks like (1102,)\n", + "log_probs looks like (1102,)\n", + "logs prob looks like torch.Size([1102])\n", + "torch.from_numpy(rewards) looks like torch.Size([1102])\n", + "rewards looks like (1165,)\n", + "log_probs looks like (1165,)\n", + "logs prob looks like torch.Size([1165])\n", + "torch.from_numpy(rewards) looks like torch.Size([1165])\n", + "rewards looks like (1264,)\n", + "log_probs looks like (1264,)\n", + "logs prob looks like torch.Size([1264])\n", + "torch.from_numpy(rewards) looks like torch.Size([1264])\n", + "rewards looks like (1346,)\n", + "log_probs looks like (1346,)\n", + "logs prob looks like torch.Size([1346])\n", + "torch.from_numpy(rewards) looks like torch.Size([1346])\n", + "rewards looks like (2848,)\n", + "log_probs looks like (2848,)\n", + "logs prob looks like torch.Size([2848])\n", + "torch.from_numpy(rewards) looks like torch.Size([2848])\n", + "rewards looks like (938,)\n", + "log_probs looks like (938,)\n", + "logs prob looks like torch.Size([938])\n", + "torch.from_numpy(rewards) looks like torch.Size([938])\n", + "rewards looks like (1069,)\n", + "log_probs looks like (1069,)\n", + "logs prob looks like torch.Size([1069])\n", + "torch.from_numpy(rewards) looks like torch.Size([1069])\n", + "rewards looks like (2588,)\n", + "log_probs looks like (2588,)\n", + "logs prob looks like torch.Size([2588])\n", + "torch.from_numpy(rewards) looks like torch.Size([2588])\n", + "rewards looks like (1461,)\n", + "log_probs looks like (1461,)\n", + "logs prob looks like torch.Size([1461])\n", + "torch.from_numpy(rewards) looks like torch.Size([1461])\n", + "rewards looks like (2153,)\n", + "log_probs looks like (2153,)\n", + "logs prob looks like torch.Size([2153])\n", + "torch.from_numpy(rewards) looks like torch.Size([2153])\n", + "rewards looks like (2312,)\n", + "log_probs looks like (2312,)\n", + "logs prob looks like torch.Size([2312])\n", + "torch.from_numpy(rewards) looks like torch.Size([2312])\n", + "rewards looks like (1636,)\n", + "log_probs looks like (1636,)\n", + "logs prob looks like torch.Size([1636])\n", + "torch.from_numpy(rewards) looks like torch.Size([1636])\n", + "rewards looks like (2019,)\n", + "log_probs looks like (2019,)\n", + "logs prob looks like torch.Size([2019])\n", + "torch.from_numpy(rewards) looks like torch.Size([2019])\n", + "rewards looks like (1450,)\n", + "log_probs looks like (1450,)\n", + "logs prob looks like torch.Size([1450])\n", + "torch.from_numpy(rewards) looks like torch.Size([1450])\n", + "rewards looks like (2105,)\n", + "log_probs looks like (2105,)\n", + "logs prob looks like torch.Size([2105])\n", + "torch.from_numpy(rewards) looks like torch.Size([2105])\n", + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vNb_tuFYhKVK" + }, + "source": [ + "### Training Result\n", + "During the training process, we recorded `avg_total_reward`, which represents the average total reward of episodes before updating the policy network.\n", + "\n", + "Theoretically, if the agent becomes better, the `avg_total_reward` will increase.\n", + "The visualization of the training process is shown below: \n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 281 + }, + "id": "wZYOI8H10SHN", + "outputId": "54840043-6fe4-4771-e8c9-78785c55aa79" + }, + "source": [ + "plt.plot(avg_total_rewards)\n", + "plt.title(\"Total Rewards\")\n", + "plt.show()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEICAYAAAC3Y/QeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOx9d9wdRb3+M7unvD1vOglJSCChhA6hKF2Rjnj1ykW8XGwXC3ivehXhooIgyrUXwJ+oKCJSFEEUkCZNlJJQQoAQQnqvb9562u78/tid2dnZmd0973vOW+f5fJL3nN09s7N7zj7znedbhlBKYWBgYGAwtmANdQcMDAwMDAYfhvwNDAwMxiAM+RsYGBiMQRjyNzAwMBiDMORvYGBgMAZhyN/AwMBgDMKQv8GYByGEEkLmDnU/+gtCyImEkHVD3Q+DkQVD/gbDFoSQbuGfSwjpE95/WPOZmhIhIeQJQkjBP+c2QsgfCSHTatW+gcFQwZC/wbAFpbSF/QOwBsDZwrbbBrErl/h9mAugBcB3B/HcIRBCMkN1boPRBUP+BiMOhJA8IeSHhJAN/r8f+tuaATwIYLowQ5hOCDmSEPJPQkgHIWQjIeR6Qkiu2vNSSjsA3AvgEKEv+xJCHiGE7CCEvEkIOdffPsc/n+W//zkhZIvwuVsJIZ/zX3+UEPIGIaSLELKCEPJJ4bgTCSHrCCFfJoRsAvArQkgjIeTXhJCdhJDXARwh3Z8vE0LW++29SQh5d7XXajD6YcjfYCTiCgBHwyPhgwEcCeArlNIeAKcD2CDMEDYAcAB8HsAkAO8A8G4An6n2pISQiQDeD2C5/74ZwCMAfgdgCoDzANxICJlPKV0JoBPAof7HjwfQTQjZz39/AoAn/ddbAJwFoA3ARwH8gBBymHDq3QBMALAHgIsAXAlgL//fqQAuFPq4D4BLABxBKW3196+q9loNRj8M+RuMRHwYwNWU0i2U0q0Avg7gAt3BlNJFlNJnKaUVSukqAD+DR75p8WNCyC4A2+ANIJ/1t58FYBWl9Fd+2y8BuBvAB/39TwI4gRCym//+D/77OfCI/hW/f/dTSt+mHp4E8DCA44TzuwCupJQWKaV9AM4FcC2ldAeldC2AHwvHOgDyAOYTQrKU0lWU0reruFaDMQJD/gYjEdMBrBber/a3KUEI2ZsQ8hdCyCZCSCeAb8Ij8bT4L0rpOAAHARgPYIa/fQ8AR/nyTgchpAPewMTI/kkAJ8Kz+p8C8AS8QecEAE9TSl2/f6cTQp71paMOAGdI/dtKKS1I179Wun4AAKV0OYDPAbgKwBZCyB2EEO29MRi7MORvMBKxAR7xMszytwGAqkztTwEsBTCPUtoG4H8BkGpPSil9FcA3ANxACCHwCPhJSmm78K+FUvpp/yNPwrPgT/Rf/x3AMRAkH0JIHt5s4bsAplJK2wE8IPVPvqaNAGYK72dJ/fwdpfRYePeIAvi/aq/VYPTDkL/BSMTtAL5CCJlMCJkE4GsAfuvv2wxgIiFknHB8Kzz9vZsQsi+AT6P/uAXAVADvBfAXAHsTQi4ghGT9f0cwXZ9S+haAPgD/Dm+Q6PT79wEEen8OnkyzFUCFEHI6gFMS+nAXgMsJIeMJITMQyFAghOxDCHmXP6gU/PO7A7heg1EKQ/4GIxHfALAQwGIArwJ40d8GSulSeIPDCl+KmQ7giwDOB9AF4OcA7uzviSmlJQA/AvBVSmkXPKI+D97MYxM8KzsvfORJANt9bZ69J36f4bfxX/AIfaffz/sSuvF1eFLPSnj+gVuFfXkA18HzT2yC54i+vB+XajDKQcxiLgYGBgZjD8byNzAwMBiDMORvYGBgMAZhyN/AwMBgDMKQv4GBgcEYxIgoEjVp0iQ6e/bsoe6GgYGBwYjCokWLtlFKJ6v2jQjynz17NhYuXDjU3TAwMDAYUSCErNbtM7KPgYGBwRiEIX8DAwODMQhD/gYGBgZjEIb8DQwMDMYgDPkbGBgYjEEY8jcwMDAYgzDkb2BgYDAGYcjfwMDAoJ+496X16ClWhrob/YIhfwMDA4N+4KU1O/G5O1/GV+9dMtRd6RcM+RsYDGM4LsVX7n0Va7b31qX9HT0lPPbG5rq0PdrRU3QAAJu7CglHDk8Y8jcwGMZ4ZV0HfvvsGvz3nS/Vpf2P3/ICPn7LQnQWynVpfySgo7eEtTuqH1ypcrnokQND/gYGIwBunXhmxdYer/16nWAE4KTvPoHjvv141Z9jiyASkBr3aHBgyN/AYBiD00qdlls1y7gCO3v7N+thd46MTO435G9gMJxBfGapF0Ub6h+7MORvYDCMwYzKuhnofrvOGJZ9xioM+RsYDGMwSaFezkXWquH+6jHSJTND/gYGwxjMmVgvnmEE5o5wIhtKkBEq+hvyNzAYxuCWf5252cg+1WOk3zFD/gYGwxiB7FMfBLLPSKeyIQAP9RyZMORvYDCMEcg+9Qr19P66bl2aH3X408vrsXxLF4DADzNCVZ+RsYC7gcFYRb2JhRGYsfzT4b/veBkAsOq6M/m2Ecr9xvI3MBgJqJ/D1/vrGPKvGrW+ZY+9sRm7+plw1h8Y8jcwGMZgFnndQz2Nw7dq8PIONZiebekq4OO3LMTFv3txwG2lRU3InxByMyFkCyFkibBtAiHkEULIW/7f8f52Qgj5MSFkOSFkMSHksFr0wcBgNIIRTL0Nc8P9yZD9Lry8Qw3a7it5FUJX7+ipQWvpUCvL/9cATpO2XQbgMUrpPACP+e8B4HQA8/x/FwH4aY36YGAw6sDJv24n8P6YUM/k2Y98j9hgUAu/DGvbHkTvcU3In1L6FIAd0uZzANziv74FwPuE7b+hHp4F0E4ImVaLfhgYjDYwuadu0T7G4cuRdA8qEvnXcrxkbVnWCCN/DaZSSjf6rzcBmOq/3h3AWuG4df62EAghFxFCFhJCFm7durWO3TQwGL5w62z581BPQ/6JTu8o+dfunrG2RpzlnwTqmS1V3SlK6U2U0gWU0gWTJ0+uU88MDIY3mMVft2gf/+9oVH3uX7wRF/1mYerjg5wHqpSAHCe8LZCBBk7YFb9texAt/3rG+W8mhEyjlG70ZZ0t/vb1AGYKx83wtxkYGEjgln+dLfPRqPlXGznD7sG8rzyIeVNa8NfPHR/aX5Ey4dwaav5BW6PD8r8PwIX+6wsB/EnY/h9+1M/RAHYJ8pCBgUEIVPi/Dq2bwm4c7B44LsXSTV2R/YMh+2Qky3/jrj5s6Oir2XlE1MTyJ4TcDuBEAJMIIesAXAngOgB3EUI+DmA1gHP9wx8AcAaA5QB6AXy0Fn0wMBiNCCz/+rRv4vwDJJW4kMnf8Y+vha3OZh2yw/fLd7+Kzr4y7r34mBqcJYyakD+l9EOaXe9WHEsBXFyL8xoYjHYEoZ71re1jMnyTLXlZ82cDZi1lH1tqi1KKerkBTIavgcEwRhwhrdrWg3lXPIC3t3YP+DyjmfvT+kuSo300mn8dHb6OS2HVyQ9gyN/AYBjDjYn2+fMrG1B2KP744roBn2c0OnwZ0g5s1cb513K2VOGziDDRu5TWLfbfkL+BwXBGjObPSKEWHDRSZZ/7F2/E7MvuR2dBXxAt7bW5bvwsoVJH2aeiyfB1KYzsY2AwFhFnkDOeqIXRPpLWo3Vcysn+xieWAwBWb+vVHp82KselFMWK3usrz46cWpK/7z2WZR/XyD4GBmMTceUX+rPQS8Vxcf/ijZHoHmcELeZy7f1v4KCrHkah7HBijHOIp709jhtP/uWI5p+u3TQoO+poH5fSuiV+GfI3MKgDlm3uwpf/sHjAWnpcqCfjhGrO8McX1+Pi372I3z63WjrPyLH8b39+DQCg5Lj8HsTd5rTfAaVAseKkbqeW9ywo7Bbe7tL6JX4Z8jcw6AcKZQelGCvxP3+zEHcuXIu1O/RyRBrw8g4KimdWbzUx+syKfH5luA7jSIrz7yt7BE3dgBjjiDgtSTuUoljWf6ey5s9ln1pE+7hq2ceEehoYDDPs+9W/4j0/eFK7v6foEVRjzh7QeeLq+fdH8x/fnAUALN8SDg8diQ5fh9JggftY8k/XXrWafw1L+/CBRdb3HVo/zd+s4Wtg0E+s3q636ntLFQAD5wUaU94hjdUrg2n7MvmPIMOfQ4yBj+t/Wp+IW7XmX8tQT53DNzog1AqG/A3GFJ5dsR2HzGxHQ3ZgFnkSev2VmQZCD5+9/SX8+ZUNXjsqy9//W43DlxFWxaX408tBPcWRJPswuJTy0Mi4/qfV/F2KeMtfK/sMHCzUU+XwNbKPgcEAsXxLF8676Vl8/c+vDdo5B2IdMuL3EG2HtV0Nb4sDxW3Prom0JeOHjy7D7xeuVe4balRcypk37h6kvT9etI/e4Rut7ZNciXNXbxkdvaXEc+tkH7eOso8hf4Mxg45eLzb8TUXFxnqhVga1yM2/eHoFjrz2UR4eWM0Aw/ozpTUfkjF01vF9r2zAY29sUe4bangx8N7ruNlPatknQfOXyzsEay3o2z/46odxyNWPxJ73H29vw/WPe/kKqmifeoV6GtnHwKBKVCOz1Cp5SmzlG/e/ASBIDKpmgGEDRda2QoSvG0Bclw5bZ7Co+ctWeei4asg/JtpHvF+UBvdloLfn/J8/x1+rZJ96lfg3lr/BmAGPDElx7IaOPvzy7yuV+8pONeSf+tCEdqINlRn5V8H+jMDyWSt0HbomKi4dVnV/xPsgRsLE9TF9tE98nL8Y6ll2KHee19LxGynvYDJ8DQxqgfQP0cd+/QKu+cvr2NJZiOwrxBCEjFoRA2vlZmFAKvusVqoiPZd1J2dbfOYA6MnTFcj/intexbcefKOKXtcezJEOeH1jvFiOuQdpB0fHTW/5Oy4VfC41JP+I5V8/2ceQv4GBAuv91ZNUckKhlJ78a2f5A5t2FXD1X17n28q+Ph2XbCaDEVU+Y4WuTSdPiZb/bc+twc+eXFF132uJrkKFv05v+afN8I13+Io+krLr8kGllhMj2cg3so+BQQ2Rhgv6fIIvlKNkUIixDmXUzPKnNNIWm4Fs6y7itB8+laq0MyOqXMYKWctay58OL9mnS6je6QgO33INZB+xto/K2hbvQ8UJNP9ahsnKq4lRWr84f0P+BmMGSc/Qls4CHn5tE4DA4lcRfZLsE3akVtlJDSiiJMAkiudW7sDSTV144NVNie2wASSXCTt8dW6MajX/bd1FrNzWk/r4atFVFCx/l/IwSydmDcb0VT0RS/6i5l8JWf61I3+5jIc4wNUahvwNDHxc+KsXcNGti0LWvoroVbMBET2livCuduwvE4Mclji1LZ/YDCOsnB12+OpkH0cR7fPF378SssBF/OCRZfjELS8AAG54fDne9d0nEvtUDUTJLWT5xzjhqwr19L9b2fEKhEM9Q5Z/DSdGclumqqeBQQ2he1Y3+87dbsG6VBF9X4Lm3y3o0rW0/OXCYrLWH+f0lPuTy1ghfVtn3Tsujfg9/rBoHX7zz9XK47sKFez08ym+89CbWFHjWYA4GHt6uB/qGUP+8m35x/JteN8Nz0TulxjnrwoPrUiyD3v74pqduKtGiXDyLMJU9TQwqAGSDMDmvFfyQSRvpeaf4GDtKYrkH53GP/3W1qSuRqDS3uVZSRrHbyD72KGZg26Qclyq1LR1fOS4NDI4phmUVNjeXcSS9btC28RoHMcNylpXI/u8sm4XXl7bEXIee20E5K+6Zkcj+3QVKrj0D4tTXFEKKCx/I/sYGKTA029txZPL1OSapM0257ycxx1COr5S80+QfcTQS/mU1/9tOS745fP4+1vbYtuQQWnUGpXDEtPkHzAJJGdbocFCd29Ulj+glkUAjxT7yk6IPLslkk0D16V4343P4Kyf/D20vVCRZR+vH3HXLl8bKwktD0qUBjOgRMtf4QtJ+l2kQdTyp9p7PVAY8jcYVbjgl8/jwpufV+7jhKQhupa8R/5iDX51tE/8Qy42Lz/MyzZ7pSV2pKj3EmoTUbKRwxLjShME/fH+5jLhR18r+1C15a/ToVk7Yl9EGS0NdvWWsef/PoC1O7xwW1GzFwdjse5NbKindFvY9yfPlMTYfUqjvgJR8y87bmSA2NZdjL2uNHCpl2C4aVfB7zsdfbIPIeQ0QsibhJDlhJDLhqofBmMHSWn+zT75r9kukn+85a9yJoqEL+9ms4KcXf2jJ+vaMtmnkVd4hq9E/irL33UpKI3WtAH0OjSzjnsFp3fc4uoqbOkKJ9aJ11kshy1/nuRVhezDZKmS44a+P1cKp5UHFNHyd/x7I2JrVzz5P/r65kTfAAXwzuv+hqO/9Zj3frSFehJCbAA3ADgdwHwAHyKEzB+KvhiMHcTwA4DA8hctuKQ4f5XFGa4BE97HrE2ZfJMgShIM/SF/MdQztF11HTHRLHIBMv4Z/+C+soOGrHeOamUfmevErF7R3xKq7dNP2UeubyT+RmRjwYmUd5Atf/1sjlKK3z2/Br98eqX2GFVfnVGo+R8JYDmldAWltATgDgDnDFFfDMYI+JRes585fDtFh29CqKeshz+3YjseXxpUwfz9orV4ac1O/p6Rv0y+SaBQaP5C33IZK5XDVyzvIEJF8IzcKq4bITq5ABkDI+FC2eFrJlQr+8izCnEWUdBY/vL3ELbow+1z8q/Q0D6XhslXNhbC0T5R2SfO8q+4FGXHTRyg5ZlkPUM9h6qq5+4AxPnPOgBHiQcQQi4CcBEAzJo1a/B6ZjBqkST7NPkO386+QKYQ48p7ihVkbMLJA/CsR3FhmH+76dlQm7/552oUyy4OnTUeQCD7VP0402hEi+jwndKaH5Dlr7o3jPBdNzqr0PXf4bKPw2c3clRNtRCjh0KzLiECquJE9XuGSGY0l32ciMwj3gf5nlSkEtgyUceSv0NRqriJfhn5axiToZ6U0psopQsopQsmT5481N0xGAVISsPP+lpGV8jyDx7W/a98CO/9yTMhAoqTG/h5hSe6FBNHHgcKGqv5T2nND8jhq9L8K4LlL5O/zsHKCLKvFFj+XVVa/nLbvSUHb/mOcnG24woO2jh9Xv7emYxUqtCIc158r1u5C/DKSURlHz35l10XpRSWfyTaZxRm+K4HMFN4P8PfZmBQNyQE+/D9u0TLX9L839zcFYrjj3M0yu0CAfnH1Z9XQa35B32b1FKl5S/LPor+sG2OGw2l1IVWcsu/7KAhE82bSAN5kLvjhTV4zw+ewt+Wbo5Y/uz2y/2JK7Ehav4hmUd2+MqWvxjn77iR5DEWwXX3onX465JNoXvqOJ7sI1dg1S4KL/RptGX4vgBgHiFkDiEkB+A8APcNUV8MxgiSatSwB1+MTlE5fJdu6uSvP3/ny3ww0JGvKA+wY2SrMgkujZZZKFZcnH7Abnjj6tPQnM+kKu3MyztEQj2jx1Y4+bsRWUU36PGaSCWH+wV0pSB0kL+nhas8n8mbm7oj0T4ODfqo6gegkH1Eh29I9gmTb9xsouxEC+2xKqv/8/tX8KnfLgof77ooVyg/JmgnmmsgYtTJPpTSCoBLADwE4A0Ad1FKB29hVYMxicDhqyZe9uCFNH9FqOeS9Z1oa/D8A88s346b/74SfSUH2zXRHiJJxJUPiAOFyqnpkXhjzkbWJihX0khQ3l+V7PPGxk585rZFwSIxgqQiDyy6c4maPxswqnX4ygMLc7pTeBm4TJ5zhRwEuapnnOYvWv5UOJXYHgD8+LG3sKuvzB3O8oAit6tb4xfwNX+F5S9LdSFHtf/50Sb7gFL6AKV0b0rpXpTSa4eqHwajE0oZQ0jgUYE9eD0hB6Oitk/ZweF7jOfvv/fIMiz4xiNYuHqHui+i7MMs/4RZiOxMpFQ9W8hY3iOcy1jpLH+/XVWc/xfuegUPvLqJr3EcWP5Rf4Mq9l/8TF/Z4a/Tyj6vrO3w5RSJrP3vg1Lv+2CO+YpgfVf8mP2dPaVI/yKyj99esSLJPm6Y0G99djUO/nqwBq/jujy6yFFo/rIVLzuISxUXZYcqZ4IMYousLybD18CgCqiIMJFwFdv6NNm886e3hd73lBxc8ruXlMe6ioc9ri/v+NZj+PZDb0a2q2YLzArO2lZEUlCBUi88kg0aYh9zNiuVEK5v41AaISndQMPj/EsO92+kcfguWb8L59zwDH7w6LLIQBOK9qk4aMp5voQ/vrgeb2wMBqq/L9+GI7/5KLZ2FVNa/mGyl0M9+bUyP41Dua/EcRWyj0z+Tvh7Z/vFeyeH58r9AfRhtQOFIX+DUQmV/p5U20feb1sExbKLjt4SfvtsuIolsz7TQGyWR/vEkP/GXQX89Im3I9tVn8nageVflK75zU1d+NvSzeE2/JIIshPRcYO2ShUXL6zageO+/Tg/r+xQ1UU58WifssNfs35TSjH7svvxnYeWRj63bqeXVb1sc3fU8i87/PPFssvJ/58rtmOTX4m14lBs7iyi7FDs6iuF+ifPArWaP6VK30dwbZTLZSrLPzo7oqHXjPTLTtQYCKSs4PPs92hW8jIwqAKqhCc2CxfJuFhxsMOXCmRubczaKFQcfPnuxfjKvUtC+6rJ0FWGelYZ7QOoI4QyPml49fnD5QpO/eFT+NivF0p98TTkrJSi6wrEVqy4+NGjb/F9jhu1/HXObSZN9ZYcTnJBspj394bHowMbI/imnB3R/MXLLlQcXoYjdF6XcqevXHRN/LxngVP+WvwtUBqN3RdRcSnyfgSTakCMC4ctO65y2U32+vvnHoJDZ7VHyk0Ao6y8g4FBvaGUfRQP9md++yIOu8bTdOXdrQ0Z7Oor88FBRDUZuiL5izp6tVCVLeaWv20pw0FVfbEIQSaS4Ut5W72lSkR+SEv+wQpoTkTiisuJYLH3jVlbGwnlUs8B36yYdZUdN8hLcKg22kf04ZQkzV8l5YTO7wbS2KV3L8ai1TtD+yOzI0nzFwcdhqKQ8W0RIuUdeH+N5m9gIMFxKRZpnKyqaBSVE/gxvxSDq8jYnDulBWt39CqTp3SF2eZNaYmeV8En1Ub7AOpwTK75+4NRktOXFQrLyrIPDSz/nqKjKFecrnx0EO1T4STHSDAuJ4Lp+o05W5sDUaq4vsPXjuwTrX1ZkhG/d9GHU3LcSHmHuAhcR5gdqSA7wWXLn303ouXP7lHOtkAga/5G9jEwUOJnT72ND/z0n/jn29sj+0pO1FHLk7wUbfWVnYjlv9+0NrgUWLE1uhqVjgSyikFBJSVUm+QFqC1/Hu3jnzcp3JNljMqWP6VBGz2lSuReyFFPSZZ/X9nlln4ay79PsPx1kUSFsoNixUWTQvapCBFJFdfVRvsUSgLxVsKhnSoDgIHlFKi+X95ejObfJ5xX5fBllr9q0DKyj4GBhLc2dwPw6p/LUFnrcdZ2b8mJTPn33a0VgDpOXU/+0QeVPc/iCl66UhNJmrPufMzyLyoGPbkvnuwT9DNre6TD2uouViL3Qi5wpyNybvkXKxGJS04UEyEm1umkq0LFQaHsoFll+QshomUnHJoqXktvWcjOTsjwFdFbqsB1k8g/fH13L1rHX/dJclPwGe98uYwFQsJtcNlnlBV2MzAYMJhBFBeeJ0JFuPmMhWLFRV/Jicgzsyc1a6tl6mQf2aIGAkK/4JfBIjM6yz9uQqAiXJtb/skrWnnts1DPgFDaGrJwKOV971GRvyT76OQlZnGL9ZE4KWsu7oePLsPP/VLHpYqrHVgKZRfFsotGjezD7ul5UnE98VrEsFFZ9hELxcnoLTme5R8n+0gx/DcKEVtiZVKR4NkMNWsz8g8+z3xRoy7Jy8BgoGDTYfa8hZNnFJo/T/IK9jELvrdciWT+Zi0LbQ1Z5bl1ln9G8aQ6NFr0S2f5xzlsVVILs9ZZf1QDlRxBYlskFOff2pABpYFDsqcYHQhl2UdnxbP+i5Z8JcHy/6EQWVRy3FjZp+yGq6iK/dGt4xsi/xiHL6X6wbe35C1NmU+w/HWDurgmgSraJ2d7so/qOx5V5R0MDGoB9kiwBziUSauSfZSWv0ckvaWo5k8IoHvWRfIXX6tkgaeWbcWCbzwKADjzoGkA9JZ/HPmrrG0mCbDzqshDjnyxCAnJU/mMjR09JV6jSG35y5q/up/sXGKJDFGOSUIcgRZ8v4zqHqtCLxmWbgyK8RWkctziwKgq08zQW6r4lr+eiMuOftbSK804gtdM9iGwCFF+x0bzNzCQwB4KlwLrO/pw2g+f4vvUDl+17AN4coD84BOiD7MT4/zPOnAaf63S/FWfcynFss1dmH3Z/VixtZvvj/NLqAY0NtPI2WHLXyY1Bpd6lqQoT1kWwbMrduBvfuRTTylK/mlWDWPLPgLhBXHERWFU2GNiU+g8ugGwp+hLJIrZlWplLYafPbUCn/rtIgBhx2s5Eu3jLe6i0th7S04oEU4FMZFLRp8g+6gtfzui+TP0Y8XPVDDkbzBiwZQLCopfP7MSb20JSLSkCvVUcAMj416F5m8Rok2tz9mB9CAeo9L8VeerOBT3vORVMX9wySa+v1rZh51PDvUsKsIJAW9QsCTNXx70uouOdtHzPSc1a/siWuyikzwp2mfm+ID8Wf0bFXp8ArUVA6wX4aO/dywijMk+FokOGK4fIqoawJnsE7f2smf5q8lftPyVGb4ZAkKI8h4Z2cfAQAIRLH/5mUtb24dr/gprV3aMqj6XsUjIIZdk+TPL0aGUy1Y6K12GihSZlZrnoZ7edYtkIxIKW/dWjPbpKYWjmeJkn99/6h04aZ/Jyr7oF3hhsk+8Jp/LWP66usFx7U2Bz4VJN6rZWEXI8I0DI//WhmxE83ep911krSgtetFLbmKoZ1rNn12LqPkTqO+RkX0MDCTY3OEbDdFLHe3jOw/7Sk4k/j/W8vfJP2tbIZkgjhwALy7ftkioQqTY9VjNX3FNuiQvMbqkIsk+ssNXXmaxp6iK8/fazdiWV0ROafkHMeti/+TyDtHPURw1ZwL2mdrqRfsIx01ozgn98ghUJcvIWb06sCUcxzVmUZLKOzi+7JPRWf7Um+Wpzs++y//3ZLR0BRCOMrrxieXY/8qHsLWrGCR5ZSxYRP0dG9nHwEACewZVpK4kfx4VFGxjFnOvQvO3YjT/DHe0kpBlJlfLjHzOJj75q/fHlRdQ68FSkpfjoqtQxrH/9zg/Jpzw5IV6ijMU0TkLeDX/RbYAACAASURBVNaxzvLPxZA/I3m21gEANGRsweEb/syGjj7sfcWDWLyuA1nbQi5jYX1HH15dt4sfM6FJIH8m+6giqly95i9fG+BFOD3y+mY89FogubmUyT4Ky79UgeNS2JrfRKNvRPzqmVXK8/YKjubXNniLAW3uLHB5LsujfaLXUC/L38T5G4xYMNnnz4s3RhbPVpITC/UUbPx81nf4KjJ8AbWVB4DHmp+7YGZIYsrFRIMAHnHZxLP8M5bt90foY5WWPxuEWPhjb8mJZCSLsg8r7yD6JuRTVhwK25LIv8Isfy9SSGVls20t+Qy2+QvbNAjlGmT56YFXN3r3zvHuS9YmWLa5G8s2B76b8SHLvxK6ZhGOG2/5sz19ZSek2//kb8v5a1bPX0X+X/3Ta7AIsGD2eO83IcUTNGTtkLQjo6+kLmsdkn00Dl+j+RsYSGAW0aLVO7FmR29oX5zsIxIse7BUmr9F9FZXcy6DV686Bf97xn7VWf4WQca3/KuVfZQOX58IW3xru6dYiQxY0VBPvS+DnUdXBjljeQOHau0A1vdWITeiIWsJ1TaDzxTKDrYLBfMyFkEuE43fHy9o/qxLKinOK++g1/zZrM5bWN7C6u29kWM8zT8s+3z7Xw8K7beFctjv3Gsi/vXwGd51JhT6Uw0MXuVT1/MbWb7DV/H9m8JuBqMWX7jrZXxXsXhJEuIyH29+ZmUkNl2VD8CISRXtQxR174N9HsnJGrBKLxZh+w+647qBwxdhi1gHlRObna/Fr3fTVahEjhNJkTt8Y25exY2uF1woO8jaHkFlbUuZrSta/gwNGVtw+AafufXZ1dguJL7ZFuFZyiJaFUl2KjJ0EqJ9xOtozNnKkh2q+j3nLpiJH513SKif7NZNbMlz/0aDIutYhIr8e0oVlCoub0P3lZgMX4NRiz++uB7XP748+UAJcc/6xl0FPLVsa2hbsNh3lGz7FEleFtHXVRGtffGYuFBAwNN2M5Zv4fltVBwa1PmPjfOP7mMzDUa43cVKZNYjkiKVnJb7TG2NtFl23EgbxXIQ6ZK11ZmorBRzi6j5Z20+ixBln+seXIp1O4OaTBmbKLOmWxuiyrTS4Zug+bM9vSWH6/NA2KHMAgdk2WevyUGlVvHeZSzCZxQNilmLiD4V+Re9yqfsugl0hoax/A3GEO5auBbzv/bX2Kl8UmhfpxTFopJ92Ov7F2+MlGAgMZq/uDkk+6Sw/G2LhCSm6x9fjsOvYevExss+k1pyePbydwfn8ztiWwRNORvdBQX5S0XOLOIRyt2ffgfuuOjoyHkqDo0kdfWUKjxHQbVkJKVB3oJI2F6VTnWS19qdgfRiW5Zy4FTV7tc5fGM1f39XX9lBQ9bG+UfNAhCW0hxK/QzicPvid+rJPhbvB2u3IZsg+5SjM43eooOSEwyqOsXQWP4GYwpfv+819JYc5fScQVco7BvvOwAAsEuKYmGHq8i/q1jBy2s7QsfHZfiGLf9gexrN37aiyTxsndsk8vdko2CbSIQt+UzI8v+vd8/zPueT7ru+9wQeXLKJ9/3wPSaEHKoMjksjkpmnlXvWbcYmkXv//Mod+MGjywAArYLsk89a2vIOIcvfIkpH67jGLN+vumaGiku1i8CIYLLPN//lQBw8Y1xokNvZU1bG8svnZrttEgziqmJzInqLatmnWHH5oKez8OtV1dOQv8GwBCMoFfnf+9J6fP7Ol7UP+/imHAgBdvWGV+Dii32kTKqyJKIVQTSWf9IKX8zyd6jaQZkU7WMTEhqQxKidloYMuoqB5s9CLlmbLApIJpmJ0gBQdt1Yyz+nCPUUNW1Rp89nxGgfebYQvLYtteyTy1hYfu3p+NKp+4SOlZFk+QPA429uQZ8g+2TscMXW+1/diLU7+jjZs9tkCz8CS7j/tk24QZEk+6gWsWdLXQayjxomyctgTIFFdMgJSABw6R8W456X1kcsdYaMTdCazygsf9/hK1r+lOJd+05RtkOgJhpCwgQajvaJf1CZ5e+4NHXxOYbuYgV2JK9AsvwF2YctMi8TtdzFp798Ei45aS5/T2l0ScveksOL4GX9JSPFdkMzEEH2YZa761LtTI1dhy5BLmNboUFOH+oZf50f/dUL6CpUOPnrsrFZP9h5wpZ/8NvMCPJd0qDPvpPxUsZyqRKEnupI3qzkZZAIx6UoVvSxxiMJ7HlTWf7HzZsEAHhzc5fysxmLoL0ppyV/0fL3YtqJUle1CFE+kPK2cLRPkuVvcfKXSblQdmIdvtu6i8hYllRLSCH7OIz8g8XGdf31jsuEyiio0FtyeE4E07fF2YF4S9p9wh/XmMWcSV7dHjEUU4yeEfsUN/CJBKwjyaK05kBeYY13Fco8Mkc/2AR+FPGv99rifRHX3I0jf1EGWzB7An8tW/46u2FYyj6EkA8SQl4jhLiEkAXSvssJIcsJIW8SQk4Vtp/mb1tOCLlsIOc3COPGx5fjnOufGepu1ATsB9+tsPxVy/iJsCyCcY1ZdEjkz7g2XOWShmK3RRBNtI+8KUT+aSx/f7m+kiRbdfSW+azk+L0nY45fRI1hW3eJy0ZBe4LsE7H8PZKT/QsqbTmp345LuezDSLUo+AVE4p3e3ogfnXcInvzSiVwycdxgdS1VBE/GIsqZ0Dv2mujtt9UDngjZ8FFdU1ehgqZsPPmz7VzeSWP5xwz6zf7v1bYIDpnZzrf3+P4ZNgPRaf7DVfZZAuD9AJ4SNxJC5gM4D8D+AE4DcCMhxCaE2ABuAHA6gPkAPuQfa1ADrN7Ri02dhaHuRmp0Fcq46Sl1LRSmgIqLgjAkRflkfPKPWP5uVPapuBS2H78e6YEmGUo+1hJIIslI45q/wvLf1VfmA9Onjt8T1/qO69DnI5p/WG4RHb6MdJJkHwCwUxSQYaTPLP9CxcWuXu8ei7OAjEVwziG7o70px++fWHVTGbtvWZH8hFXXnYmpbQ28zaD/UjSOvy+ydKfiOruKFe6cFWWfIwWLnG1nJB8if1Hzt9JZ/kwGa8ra+PBRs3Ds3EmwSDTOX8fxw1L2oZS+QSlVZeecA+AOSmmRUroSwHIAR/r/llNKV1BKSwDu8I81qAH6yk6q+ibDBY+/uRXffGCpcl+c7JO0KIhNCMY1ZTkxMYiyT7Hi4LQfPoUVW3tgEzVpE+hkH+l8QvRHUkx2UNsnSv4dvSX+/bGMT4asIEWITmiRFFsl2adRI/uorinJ8geCctQs6ue5Fdtx8NUP469LNoWig8KWsvfadQOHr9Lyt9ULmQTtiJp/mLbY9cjkr7uiBsHhy3D5GfsGnyNhrT808AgzLzFkVyUxMbAcjIacjfamHH77iaNw4O7j0COFeuri/EdabZ/dAYgLaa7ztwHAWmn7UaoGCCEXAbgIAGbNmlWHLo4+FPya4yMFcTH87AFTOXyTBjhbY/k73OELbOksYukmz2fglWVWk7xa9lFb/paVrM96FTW9aB9Z5ujoK3OpxrZI6Luc2JzHps4CMpLDV3a0hix/5vCNkL+6X0lgmj8bBJ5Z7tXIv/QPr2BSa54fp5JoKq7L+9GikO1sjezD2wkRcHgfux2y7KMbiJlEI0o1ogTEru8IfzYQsfxDSV5+mzGWf3Pe+06bhHDQplwGvb7lnx+umj8h5FFCyBLFv7pa7JTSmyilCyilCyZPnlzPU40aFCrREgXDGXEkziwqleafFNJnWwTtPvmH1q8VQj13CHVlLIso4/mJpqSzzuGrm0GIYA7b5Vu6sXJbuADbJ29dhFV+zRlWBoJhYkuOb7dDMwJR88/CcSk6esuwiLhwjCz79Nfy98iLlcFmklxnoRIqJieHRgLA1u4iFq7agYxFeBSSfP5Y8pcSrVTXIzt8dQazmKmsar+tIYu/fPZY/Oi8Q/2+hVc9Cwb75GifWz52JBqz3vWKmcXN+Qx6il5tn2xCtE+9krwSLX9K6cn9aHc9gJnC+xn+NsRsNxgg+krx0SLDDXHkz+LGu/qh+dsWQVtjFhWXorfkcO1bTPLa0RsuKqZ67iyiTvKSNzHyF4lBB1bYTUxwEvH3t7ySFDYhoESw/FvyfLs4KIhW4b67eaUa7n5xHXKZYJ0BebBU9XG/aW0AvEiqp9/apuwbl338QWCNojgau0b59Xk3PYuO3jLyGStEguJ1sLDQxDZtmfy9v2llH7YOryj72H69o4pf++iA3ceF+ia+ZrWYRIdvXkH+TTkbJ+w9Gb9f6IkdYiJYc96rL0QIUmj+w9Phq8N9AM4jhOQJIXMAzAPwPIAXAMwjhMwhhOTgOYXvq1Mfxhz6ytFqjMMZuoGKUsrL96qSY3TLATJ4RcKC+vaq823vDlv+KgtfV9hNJk/2cNopyF+O05fBumhLmv+k5sDyFyGuOnXiPpNx4O7jsKuvzOvuA9H7pUpc229aG17+2nvwb0fMjO70waN9fPnnbWHtYREqzb/D97+UHVdJlBmL4Jr3HRBK5gq3GZ1N8Pf+OeSsZB1Uso84y5O/8lCcvxDe6Vn+4TZFnLiPp1iwwU4c9Nobs9jZW0I5TYbvcCR/Qsi/EELWAXgHgPsJIQ8BAKX0NQB3AXgdwF8BXEwpdSilFQCXAHgIwBsA7vKPNagBCopFOGqBXz+zErMvuz92Wt4fyJY/k2iKlWBh7f5q/vLKVkA4ykes/6/T/FOHejLyJ/qMYPFcqlBFlmTFpBQ5x2DG+EYAiph9oS1CCOb7FnwuY/PzyDMl3eDjRefoL4DJPczy18lvcVY6Ww1Lhm1ZGNeYxUXH75ncptRHtidi+WuuUyX7MF8Mey1C7G8ox0LQ/OVTPfz54/H9c718BuZgFjX/iS15dBUq6Ck5/Leqk3eGZagnpfQeSukMSmmeUjqVUnqqsO9aSulelNJ9KKUPCtsfoJTu7e+7diDnNwijzy9LLK9IpcKhVz+Mq+5LN+5+/5FlvP1aQrZIGbH1CNY+0/wffm0TfvXMSu9zKciflQeWi5oxiEXcLI1WL6byy9vD5/O3p7H8NccctafnXNzV5y9YYof7tLcv6aySpBZZq29r9CSufCZIRpKjo+JkhLg1iINon+QSFgxJ94O1GayMFp94Bagcvupz6M7M11+2xdlE0O+4+yPeHtuy+LMmP3JzJ7dw0mdyT4OimiibpXnnVZ9zWIZ6GgRwXIq1O9Qa6GCBLVGXRvnZ2VvGr/+xKlW7/Iddgx/hss1d+PRvF0UWzwaAw655BFs6C6E6MWzpvotuXYSv//l1f4Hv+AsUSwWEZB/BMBTJX2v5Q22lauP8NRnB4b5ZymPGNWaRsy1ej0i2/FnpZbnyqEz+LIaekIDcrv7L63hxzU5+TFzh0bgM5UD2ia9jI34/SYXuxOimOOgS2+I+m+zwDUtJQeZuun7YJDAo9pzcjE8Ksxbxd8MWehFlH7GeUpDhq5F9hiraxyAdfvjoMhz37ceHdABg5J9EjtWWgOBaeQ0UpS/9YTEeXLIJr23YFbHgOwsVPPLGZk74QHRFrjc2dipryYsQlykUj9VZ/jpr3CJBBcfw9vB7Me47jeWv6n9j1kZLQ4aXobYlJ/RsKdtXPjcDK+ZWrLihgeHLf1gs9D/G8o8hGp7kJWj2H3nnbOw5Odw3UX5JS+pJpbBFwpe/E/0pdLKPtz0XkswCv0Jcnz1r339tW6HVxS4/Yz/lZxoUA5y4jkBSbZ9hKfsYBHjiTS9KQwwhHEyIhcJki3rxuo6QFCTHv6dpG4hfaKRaUKgHqaxtoacYLLJdctxQ319YtTOF5W/xB1tcACVE/l3B96TLzCVE7WzThXp6cf6xXUPGInyQFtGYs3ltHnaceB5mpR46qz30OXkW0uZHzBTKToj8xWuPkzVEy/+mCw7Hdz94MH8f1PYJLNiDZozDV84MSO8Tx87hfgcgjdXqk3/CDEGur3Pjhw/D/tO982hlH82pGdmGi8UFMlms7CMl2LH7yr6rD/rLOopgPhKxWRa6CyQXhatXqKch/yqwvbuIC29+PrT8HANfiDnhi6wXxEgH8UF/ZW0H3nv9M6EKmJ1Vkj9rrhbZw+x3TKm6vZxtcc1/fFMOpYobGqyWbeoKfe7k/aZg0VfC0ciWBY3sIzh8JctfV95BHecvvw9kn6SwPNsiKJTVln+zkPyk8h8sveY0/P6T74htv82XfYplN0SY4p2OIxPRAj9sj/F8jVpAkH2E3/i4xmxowZWvnDU/4hSVP6M8r/CZfzl0d1x++r7a/TYhOOPAafj3o/eIvR7dZapkn+a8zduJuz+hBDsx8sff/O1/PQirrjszfD5+7aLlHyTFJcX5j7RQz1GJW/65Gk8u24rf/HN1ZB+LKhF/UKu392DTrsGptSOSv0hyW/yoFpFAq7b8uVOrlpFE6vrrWdtCb4mRfxaliosNHcE97ClVQp8jhGBiSz70wGUsIczRdbFiazeu/vPrITlCnKHJJRMYdA5f+WHklr/meBEZmyhDEhtzdqj6o62QnBqydmLVUGb5lxw31M/wovX6z4uhoznJwctkH7EPbY3Z0KAlQ1UZUwTri7j/B/92CD55wl6h40JJXqz2DifrKjV/vi5BcEBzLsPvV5zM4sX5B6+Z5DW+KeefU/9ZcVe7kNMwVFU961XeYcwh0KaDh+yE7zwBABFLoB7oC1n+wfbuokf0YtRLR2915K8qhdxfsAeAUihzErI24eGd7U05rNnRiw0dQUJUbylcv0g1exAt/1KF4uLfvYQ3NnZq+yQXS+N9RTiUUmw/9F7I+EwO9bRC1TAZGjI2LwMA6GcjSWDRPjLCUU8x/ROul8kjbQ1ZFMpFpfXe1pCNjf5hFrs84F1//qHoKVbwvYeXRc6rbieckAUgFVmrwDT/ULSP4GOJ1/xJ6PUVZ+6H98yfGkoKk8GMJrFVcXbEBiHdZRjZZ5iDWZUJvsi6IST7CE93t6+fiwtdiJb/q+t2adtcsn4XlqzfVVPZh6Hs6FdeYpZ/u2/5s0qle0xsQk+xgorrcsJR9SljWcj5WZybOvtiiR/QJ16lDfVkD2eqwm4WQUGYgezW1oCnLz0JlkXQIlS7TOM8VqFNUTETCDth42ZwWQX5s8zbvILk2xozsZY/Izn5azrroOn4tyNmpSJceT8bBywST5q6Qmk5heyTpj3xGNanfMbGcfPiy8/ocgE+f/LeAICZE5oibevOWUsYy78KcL1asa9USY606SyU0ZRi6t4f9JXUmawsTl6M9RbJ/+zr/46nLz2J/wBFnPWTv4feJ1RVSAV2D0uOqyzT4LgUPX6o5/imHEqOywe2yS15bvk35zIolEvK++2l6vtZqFu8mjMTm3PYrnHGe6St6CtJV9hNLO+QJPvYFgnlS1BQfu9bRMtfk3uQBFXFTADY0RP4OOIXTQlbw0AgJeXsaIjnuMZsLDmlqRmU5jhVkpeVMHAkh3qG5SN2fKLsQ4PyDmkQWP7h4//75Hm4+KS9gv7rLH8T6jk0eHbF9lDSkQ5M89dl2FJKcdBVD+PSuxcr9w8UfRqHL5N9RMdnpNplSou+FtnDzDL+8WNv4a+vbYrsL7teaQdCPNIpO5Rbre1NOa75N+XV5YoBz5JnDzjL5NWFSgJ6K5to4vblTUHd92QrLWMTFIRQW9Eibwk5fPtn8TUriqYBYcs7rjqGSn5hlr8ySilrxzpzZWK++SOhNZ84IdoJepmqsBu31DWf0W1nGjv7jTBfBjs+bgAXv5O0pHzIrPEAghXoRGRsK1G+MrLPIKKv5OCel9Zhe3cR5930LP7r9pcACA+9ggSZZa0jUiZx/PHF+tSxC5G/YFAzy7+isfyB9OH7NdH8/b+LVu/E2h3R4maO66Kn6KA5l+GkwmSgcY1Z9BYdVBzKSU5n+TPZh8Xz76GY2TDpSF4dK9RWGstfTPJKeKLEBUDmTmnB94RQSlE+yVhWvzI70xBSvOwTvQBWME4VyUYEqUtVsE2cSZx50DS8a9+p0ufZcVVo/pJGXr3mL5F/NhxtE9dcNau2MRwysx1Lvn4qTtl/t9jjBru8g5F9FLj6L6/j9ufX4PrzvZKur8To4gxJsfBJiUkDhSglOJRiS1cBb2/p4Zp/WaP5A8lVMhlqUTQu6Xdcdih6SxU05WyuzfYUHdgWQWtDxrf83WChEsX9toUMXxbttMdEz/Kf0JzjkT6TWvJYt7PPr+qZnvzlLdVV9QxI7KHPHR+ux18Dyx8AfvyhQ7HXZP1MJ24GpyK0z79nb+w/fRyOV1iuDHdcdLRSOhQHQ1XxM25tV6H5s8F2z0ktAIAj50zACqk8NpBcz5/NJvjMJZXsE7yuRo5RrWEgg82CcnZ4VbM6cf/Ytfwv/+NiPP7mFuW+9X50CbOamZ7Pvhz50fnF0yv4ax1Bsmigek3hZIfvB376D3zo588Gso8gL8g18tOOS4NRMJRp/s35DLc0u4sVZG2C5ryNvpJn+ccVFxPJf2tXEYQEhdHEEDtWJtnSJHmxtmREZB/R8k+h+e8xsUnZdrMU6tlf8n/vwdOx/3R99Ems5q8g6Kxt4cyDpsU6s4/ecyJ2b2+MtieGjsb4uqrR/Nl9O3DGODz1pZPw8WPnxH5WBivpnJNkH3E5Th3E7ySt5Z8WrDlZRhuWVT1HKiiluP35tfjor16I7Lv12dV4ZrlXz5xZlUyX1X0H37j/Df5a92Ax8h9ozO7LazuU03ZZ82eSCgvrFElS1m7Tav4Difb5x/Jt+MviDdoIDIaK46K36Fn+WW75V5C1LTTlMqi4XpQQGxjEwZatw2qRwKrb2l1EW0MW45s90hdLRDf7swddbR8gePD2m9aGE/ZWR3WEHL4pMnzv/vQ78aeLj4nsE6s+plkPOA1WfuuMiM4e9zXGFXYTcfen34m7EhLOgPDvnZGuCJKCcAFJ8xeOnTWxSTlgnXXQNPz03w/j7w+eEQyGsuzTwGUfhP6qIMp2tY6/Z/dCjqoy5R1qCJHEZCL96r1L+H5mLctri4ofkWvP6GQf1sZAsvUeX7oF77vhGdz23JrIvpDsI1wfi3ARo30KZQcHCnHJaR25A3H4nv+L53DJ715KPK7iUvSUKhHLP2dbIXJk1pF4rTd/9Ag89LnjQUhQz99xKdqbstxp2VOs4Bf/sQAXn7RXEJ8fE1nDpvZ7TGjCkXMmKI+pZg1f2yKY1JLHwTPbI/vEFa5IirbSgBAS0fHj5LukMgsMh+8xXns/wu0J5B9n+VcR5y9/VyoL/PrzD8NBM4J7/KdLjo30g52TL6YCpvnHR/vw1zUmZdacPEMy5F9DiFawbkUlIFg8PI7zOvrC4YO8Bpr0oVrIPqu2e7rmW5u7IvvECBKXBhYcc3iKPofekoOpbXn88kLPIqy15e+4FPe8tE5NMgnXX3Go7/C1+UPZW3KQy1ihSJacgvxb8hns4zsnRaJpb8px8u8tOTh5/lR86dR9ObFnbPViLkBQAZOCxiThBNZrmpW8dGjOhR2mtTIsZUKPc9zXWsoQyVKp+RN2XPraProM67QQDQNAiPbhfYkhf1H2STlLSgv225Ed6ynH4+rPV59mhzdES365ZjUiQL2QiAw5W5bPGgRLu7dUwfIt3nnSjOI7e0pKaSduECqUwrIP+4Fz2Ue45r6yg4aszQkvbRRPWsv/tudW4/N3voLfPR+doSSBWf5N+QzPfOSyjxAHP3dKC2ZOaMQVZ6orKYbIvzHLY9VFiOUB0pTTTUrCSRvnr0OT5BSslcUnSzlxY3itY8rttJZ/FZp/tZ+Vwe4HW/OXyT5ByKX+s+IyjkkDVrVg55VlLBPtU0OIYY/lmNWpOqX1Y9lXQAWX704pcYgRqZhR+4lbFuIfb28HkDxV7CyUceg1j+BTJ+yFy6TiVrwfijb6pNo+uYzFk6UAL36eoVBy0Ji1+UMjW+irt/fg/J8/FzlHWvLf5kfYyLXnASTGlXqaf9jy7y5WkJcs/5Z8Bk9f+i5tOyLhtTdllZmv7Lvwauyr2wmt4pTgFLZJcmRGnKTQFLH8a/PQywNObWs0xSNJ9kmb4Rs3KImf/fJp+3KHOuCtkNYo3Vd2/G7jPIf/8X6GLmsl7jsKrZ9cc9nHa0+XeVxrjFHyD4hZtNArkrbfKVn+Kotpp2T5u9xfEBzMiB9IJgdGnP/vybc5+RcrDnYl1ONRkb8I2fJvzNn8xytHzNz6z9U84klE6mhVv10Vx5QTwkq55Z/L8IzSnmIFLfmGiEM0vgsEWZug7FC0N2ZDZYjFY7y2ktdPpVRfLkCUfQbiBJTJn8QYlh88fIbymlSQyaSWZTqSkOjw5SWd+3/fRFlL9kV8UbEmMPuu505pxdOXnsQjwVgz6mJ+3m9ArORZe4ev91d+dusV6jkmyV+0gpmFvmT9Lnzv4TdDx8myD4uHF5+djl7J8mfkryG5pGm1OJi4LoVlEXzhzldw/6sbQ3XTZYjlHSiN/oDEQa6v7Fn+lsby1/3Y0lr+7BJ/9NhbeG1DJ35xYRBtkrQOcNlx0Vty0JwPLP+eoq/5h5Kgkp8Ib+EUqpR8vP3RPlsk/P2GtWb/rzQIiMv/DcRKk7Nz49r6jpAclgTVOrqDhbSa/0DkJrH4nlgcLw3E3IRAvoseR+BNWlWL09cK7Px5e3DIf0xq/qLUw0jxrJ/8HY/7C7IwdEmyD7OQRSs6YvlTpvlryD/hmxQHE5a4wsogyFFHIkIlnQXNn4H1x3UpCmUvSYr9eGXNX2cFx0WJbO0q4op7XkWh7ITI8dE3NoeO69WsA/z+Q3cHAGzaVYDjUkxta+DSTclxI9E+afRWNtAwyefWjx+JR79wPN/PvgvH1ZceVmn+8u0Ro30GwgdN+cFx+NYiWS8txFlHnOY/kEFTNATSJFPpwFrRlfrQ7asVWMtiHsLNH1nAHdK1xpgkf1GPrzguPneHOgQxavlHnblRy5+1GxwjPsRJPx5xMGHRRozI2SIgqiZk2Uf+wbD+sKigxqzN+/L5O1/Gz58KEtV0PYxzDH/hrpdx23NrsHDVsqRhuQAAIABJREFUzlhLRR5QGT64YCaaczbe9ge8mROaQrMXFufPkCbSgnEcK3Z23LzJmDulle8PKk7SUNiniDD5q88TivYZAGPXK8Qv6vBNJn9VqYb+QBywlZq//3cgfgjxO4qrMJoIPrjH+xdo+PCagUf72Ba++8GD8dj/nBAph1HT89Wt5WEMkbyLFRf3vrxBeZxIVJRS/jlx8Ngpk7//I9ZZ6TpuoJSis1AODSby6mCsDjyzrNds78VV970Gx6XhSpE0qvkzy58dJ1r+27pLuPaBIFFNx/5xBuNzK3b47Vqx0Zy6CCqml6/0o6/2mNAUynTMZsLRPtVoxDrZhz1sLqXaZKM0hbyqKe8QB5l0akUu8jUlkf8jnz8eT116Uk3OLV5TXALZQOYiokY/EMuf3Sbd2g5AWCqs9RxAdH7/6+EzlOUyaokxSf6iVb6ly6sV/5750RG2sy8gqkI5KEEsfn5nbxn77taK75/rabDMgtEtnqEjh9ufX4uDrno4tNwiI/8st/zDkslnb38Rv/7HKizd1Im+shPIOAqHL/NzsBlCQ9bWRivoHJs6ucBxKR/sihU3lrSKGs3ftrzr7Cx4FT13H98YshRzNkFTNr3DV4Suxj1rwrP8/XZjLH+dRRiUd9BHgDRmbZ5XkRa1s/wl2SeBaedNbcXk1nz8QVWAWf+qwnC1KG8uDspJy0XGthMT6sl9E/WUfVJmO9cKA7rzhJDvEEKWEkIWE0LuIYS0C/suJ4QsJ4S8SQg5Vdh+mr9tOSHksoGcv78QrXK2ROAsxSgrSim9pQonUDE6Znt3EZNa8jjML9saSEPVWf5PLfP8Day0hNgG+0HLa78yK9pbF9bhSUIOpZGHgPkp2ADSmLW1PzLd71sXJSLOfsoO7ZdD0SJBpMz0cY3IZ+yI7JOxLX5daTNRAX2N+5DmL8Tqh49RvxYhWv66ezdvagvevV91U/haEY08S6pFae5qwAZflexz0wWH42PHzMGcifpCdNVgIFnR7JPK9ZwREHO9QmW5ATISyB/AIwAOoJQeBGAZgMsBgBAyH8B5APYHcBqAGwkhNiHEBnADgNMBzAfwIf/YQYXosGUhjWJssAq9JQcOk30cF6f84Emv7HNPCRNbciGrG4ghf80XyzJQRc1/XUcfdvSU+HSZ6fXst8mIvFD2FjxhU15Kw/pxa0OGS1bM4RpL/pp7oC1dIVjzpYqbGNGjgliMjRUIyymchUzTrcryr0L2icpJ3nsKPRmHCrtVeU/jUDOH7xCGegLB0pKqaJ89J7fga2fPr3lyWX+QKsPXL88BxDuw+3V+fo7BEWQGdBZK6cOUUqaNPAtghv/6HAB3UEqLlNKVAJYDONL/t5xSuoJSWgJwh39s3UEpBaUUa3f04u4X1/HtbH3YJH2tp1Th4ZvLt3Zj2eZuXHHPEmzrKmJicz7kPATU1SYB/Q9rXFNAUGwg+Oq9S3DYNY9oZR8mofSWKugrOzxDVJZ9xjflIpp/U87WFiHTGU86i6cUip5yY6OSdBAtf2apZ4VrYNfTJBRjSwu95e/9pYLsI5MQ4cfEOHxZtE+KDN9qUIvaPkD0Xg2y4c8t/1qXQ6g1YjN8hYHhhg8fhm+9/8DYBYL6A74o0CDdplrG+X8MwJ3+693hDQYM6/xtALBW2n6UqjFCyEUALgKAWbNmDbhz77vhGdgWwZodfaHM0427/PVhBfK/4oz98NzKHaEwRU/z956aJeu9NWH33a0VL67pwKTWHH/oGe/pMod11mObQFBT2/KhmvtZOdqH98kj8r6Sg76Sg93GeRazbKGPb8pyJzXX/HN21bKCjtNFsldZ/mlCC22LcHJoYBqxyvL3I35UC6vroCP/0w/cDb9ftA6HzByPx5d6sptM3uK7pESwuNLQQ4lonP9gW/4e+ZfjlhAbBLzytVNiI9bYXVKGevp/beJZ/h86cuCcFDmHRnqsFxLJnxDyKADVEjRXUEr/5B9zBYAKgNtq1TFK6U0AbgKABQsWDPhXo1uQxXEpWvMZTGjO8W3HzJ2EpZvCxdMqjhty4gKBBDGpOc+tP/bjKmsILw3fzpnUjGWbg5pDzOqVSzEXuOXvoFB2+RqwlFKhaJWFhqzNH7xUmr9GpEgl+zhuxKmbZiZgW4RbqKxWf05y+ALgqfrVWP66OOl37TsVK791hpecJVjvOiTV/0lT1TMJt378yEjuyECRlQu7Dbbs4w++nX21va5qIc6uVbBiyLcWyWhJYC3XurieDonkTyk9OW4/IeQjAM4C8G4a6ALrAcwUDpvhb0PM9iHD5NZ8KFU+n7UiqfZlh4ZCPAFgS6c3g5jYElj+QXkHNeHpvtaSMLAcNKMdD70WzDoYCTLJhv0Q2UPcXayg5Lg8Dt5xg4iOFr80MssZ6CtH4/wjfaxW9pEs/2JFkqfKacnfu87GnF9f3QpKNASaPyP/2uiicvJOPPlrtgsJYtrPpxwUjpunXjNgIJAJa7At/9MO2A33vrwBe09tTT54CMF+UmrVJxjg64W0dY5qhYFG+5wG4FIA76WU9gq77gNwHiEkTwiZA2AegOcBvABgHiFkDiEkB88pfN9A+pAGslYuo7UhE4qOydlWtDaOG7X8N3V6ktHElnzE4SsPFAw6o0t0EB88I1zrnZc5KAWhpyIRs2UJebSPS/kD3tKQQcYivO+9ijh/GTrLMGmhGnYdsuwjDwYq2CSQfcQEIzbwZbnm7w1w9Sqn2y/LXxhAhqPsI4NFpg0WTjtgGl696hQcOEO/ulgtIM7e+4O4hYYGw/JnhmM9w0lFDFTzvx5AHsAjvgX1LKX0U5TS1wghdwF4HZ4cdDGl1AEAQsglAB4CYAO4mVL62gD7kIgdUuVNGfmsHZqu57NWJFSy7LgRJy7T5Sc25yIO35JG32SDwktrdmLG+CYeTy1GIIkLrQBBckyPsAqVWLFze7dP/r4M5VKB/PMZZGwLZcfFI69vxlu+nNSQtXnSWLSP/Sd/leavK+kgQrSaxVlYe1MOPaU+PgiwAS6NdXT9+YcmfvcMyWF2+nr+QWmIwdNr+4vvffBgnHHgtEE/b6sm16JWePrSk7T5HGnBnfuqff7fen697PkaNrJPHCilc2P2XQvgWsX2BwA8MJDzpkWx4uBLv1+MY2MWngaiiSH5jB3RicsO1ZLfuKZsKGwQiFYIZWAW+L/c+A9Mbcvjuf89mbcPACfsPZmHxjEEyxl6JOq44VLSLCuYSVUe+Xv7WvIZ5GwLFZfiP3+zkH+mKWdrw1F123VyQVK0T3cxnNX7sWPmYOOuPjy4ZBPfZlmE91kk/3GNWazv6AuiffwBLs0DctZB0xOP4ecXwjWTjtFtj8vwHS5DwgG7j4uUOB4pmD+tTTujrkU2LDMAVfIm3zfgs+jBjK5h4/AdydjSWcQLq3bgvlfU5RsashYKZTdC9PmMQvZxqJYUc7bFw+d4tI+O/IUBZHNnEHVUclxMasnjlo8dGfkMkzwYibqUhqKB2GtGmo5L+RTy4pPm4u4X14UIOuPH1OuITpa3GHSSVVK0T49E/vtOa8XcKS0h8rcJ4QOmTP5AMPupxvKvBrpIC3FGmGj5jwDZp8ah6YOKB/77uLq2z746lY1z44cPw8+eehstufpRJjOu6ulXEDGCfwrJmDmhCTddEKTUz5vSEtrPtOUGacHknB2VfSquq7X8s7bFnUVbugqYfdn9+JOmXpBIzCLKFZdHtMiQk2Mqrhuyphn5sz4z2eeI2eNx/N6TkbEs9Ar+AnbdOgtDZ12lkn0cGiX/Upj8c7YV0ewtKxh0RM2fzYLY7Idr/jVOhOGVOWOa1Vv+wf7B0mv7i+Hev6GEWO5DxvF7T8Ztnzi6rlY5s6FGhMN3JGD/6W389R5SCjkjEtnytywSsfxLFVcZvsm0ajZaszIN4gIuIiqaJKiy44aSmuLguDRUIC0gf1/2ccNZq1mbhI5vSLCe9ZZ/Ss3fkS3/sOY/d0pLpMiXTQhPomPRPoCQIOT3lUX71KuWus7qikvyIoTgo8fMxon7TB725DpYxDISYXHZZ2jOz2qHGdmnRhBv5EQpGoBZ/PlslHSjlj/lX44IRmLsoXp7q1eSuCWfiWjdrB01+VNturhscVcciu4ik3osvtwkux6HUrhuELqWta1Q7D2zrLWyj8bCF2csBb+QXNa2QtdTdtxIaGevZPnvP72NL0bPkLEsLpU1ZETLP1h4HQAac+k1/2qQJswuLob/yrP3B6CfHQ2XMWG4D05DCRJj+Q8GuOVvZJ/a4c6LjsYXT9k7Ys0zy79BkQSkWgZRZREzwiYkXNRLlpJ4O25UFgE83VxH/rL/QLT8p7Q2BJa/T+o7ekoouy4nMlliYY5hnXKi81eIl3/uz/6J7/orn7HrachaSsu/W7D8D9i9DYSQiGwjyj4Nuajmzwa4A3cfh3lTWmpaddKDOtSTnX96e2Mq4tTODgbWuZrBWP5xqL9TNw4sidLIPjXEUXtOxCXvmhchwXjLX5XkFf1ZiHq8OGLLFTgZHJcqyxqXHb3mL+cpOFQk/zw/F3OUXvfgUry0poOTlTyosON0urlW9hGuf0NHH970s6AZ2bfkM8o4f+bwvers+bj70+/0+yTJPhbhEU8hzd/PDmUD3CEz2/HIF04Y2KIdCuhK9h45ZwJuOP8wXHHmfqmcubWqx1MvGPLXI6j1NDTnZ8+XIf864Ji9wiGffM1M38o/88BpaPdTwOUBoey4ofBNNjMQiVWUmHSJZY7G8i/HWP5yWYeKb/lnbYL2pkDKkqUqXaVK7vDV/Mb0CWrhRXBYtBK7nuZ8RlnegZH/pNY8H1TlSpMWIYHsI5D/yf46Cx84bAYGA6oH78yDpqEhJiM6DYbLoGBkHz0CzX+IZB9j+dcPJ8+fise/eCJ/z8iMEdINHz4ML3/tFG9bJMImbPnzEMRM8EWJ35k8S/jP4+bgk8fvCSCqgQNAuaLX/OVZhONr/q0N2VAZigZp6T1mXMvtsjhvHSHpCnCJtX3KjostfoZzyV/ApTFr+6GeDj581Cw88F9eaB7zfYgzjawVtfzZ4Cpa/jPGN2HVdWfi0DpnpbLexGv+de3CoMBY/nowg2+ovmdnkC3/Ue/wlTFtXAN/zW62Sp9XW/5h8t/aVQwRa5yjpr0px39UqozXkuOiLafOUJRnERWXortQQUs+E+p7QyZqTQNR/0XS+qxay18Y0EoVF4VyiWv8rCRGseKiUHbRnM/wuvzsekWpR45o8KJ99N/HYKE/5R1GEgbLmTgSceXZ+2NKax4nV7noTq3Ayd84fOsDURphXK6q+pizw0sGyuUd2n3LP6eRfaLtWby6okj+n739Jb/tKjR/10WXT/5i3/MSqTPLXi5pLM8QZGgtf744vcsTvpZs2IUXVu1ALmMha1t4ctlWlBwXFiF8AOWWv3Cv5Cu1NJb/YIFH+/QjwzdV+/383CUnzcVPPnRov88rY5DWCRmRmNCcwxVnzq/J0pL9wWBb/mPupyBKHcySVa37KVr+rDCaGOrJZB/Rqpa/NFFrz9hB7Zo+Qfb58ysb8PqGzupkHwp0FSuRgnTydbDm5EWtcxn1j2v2ZfdjQ0eftjQFk8nEweEr9yzBS2s6vKxoof8VfzYAAG/7i7KLUo9KcuLlHYag/ADRRPuIGArF5Iun7oOzD05fpiIJRvYZvjDkP4gIZJ8o2chVPssODck+bY3RdUlly1C0YDNCVqss+3QVKig7rtbiKFTUln9rQyY0SMnXYXHLPywnxfmz/vH2dmVUEyFC0TrBofvmZi/ix7YIT1KbNaEJnzpxLz4TWuHnPmRC90rfB1Xo7WChv3H+Iq48ez7+dPExtepSTTEapKvRCmZIDpbsOeY0fxGBwzd6s0WLPmMTr6SzSzF/WhumtzcESw0KUo2cHNLakEGXL3nkbALGmTL57+or+3H+6WSfCnf4toZlH43mL1v+cSiUHaXsYxPCB0sxjp9t6y06fCZ13pEz+TqnV549H1//8+teGwmW/6wJTVizo1d7H+qJNCV70xpkHz1mTg16VB8MVsVIg+rxuZPnIZ+x8P5Bimwzlj+S4/xZSeSK6+KEfSbjFxceEdSZF6xZObtTtHQzlsUfPDnaZ2tXwY/zTyn7+KGenuYfY/lLa+IyxFn+hbKjlH3EqpuqDOWuYgVdfiLWpOYgAetfDw9+yCGHr4KD/vCpd+C3Hz9qSMMi48hxIGn3w8XgNrLP8EVrQxaXnrZvzReG12FMW/4sdFEl++RUso9LOTkwYs/Fkb9FeOXQbMbyVjZA1PLf0lWMLe8go+xSdPaV0dYok79s+Xt/dWvYqlCsRNctYG2pZB8RLBFrUmuQeyDOOsRQTxXBT2lrwJS2hsj2wQDrTZwsMhpoc7jkGxgMPcY0+cc6fCXZp1hxQWlAYMyKjbP8bYtgXGMWhXIRWYuA2szhGyb/rV1FlCv6JC8ZXX1luNRzOosRPnLUkq2RfWhMAnux7CjLO4iyj678Q6efdcwkHyBMNqLlv+9urdhvWhtmT2zCho4+bX8GC6yfcdxoiNNgNGFMk78jJXmJCEXqWARbu7yEpnGN4ZLCtkBoOvLf3OnlA7DzqSz/kuOGEsbOOWS6tiz0dn8hl3GN2XCegaYWfUu1lr9C87csQfP3Lf+MRUKzBLZA98QWdd0dUQZryNp4sM712fuDuKX82O3tzyAQ166BwVBgTGr+/3ncHHz82DlgkZsq73rYYrWwaZdH/pNbPVmCRe6IMoFDZc2fBCWJbcIHFLm+vSf7hDX/H5x7CJ4QspFFMHmlrSGrHLgYGFnJx8QtEl4oO8okL9siPO2dlW+Y1h6WaNhAIFdPZRgJzsY4XjeRMgajCWOS/K84cz6+etb8YM3MBLkla1vY4JP/pJacv82XCYTjZEeqbVk8H6BUcflsQZZ9tncX4dJonaCmfHzI47jGrFKyCs4fJavFV50SGzderLgoOxTv3ncKvvkvBwb9IYQPbkz2mTk+vHTeZ07cC4A+iWywHFkDQRy9G/I3GE0Y/k9jHcHILMkizdiESx2T/FLCaVaSyliE5wN0FipcIpJlHybjyOSYt+PJv60xq4xUYlDJE0mLXLNon9mTmvH+w3bn2y1CeIYvuxfi4jjTxzXg0tP2xarrzoy0qSstPZxASPhv3DH9O8EAPlsDXHX2fBwys31oO2EwrDCmNf9PHDsH33pwKbfOdcgKRM+cmdzyj3mobYvghL0n456X1mPWhCYU/WQt2fJnMxA5vl2OOJJDLMc1ZpV1ghj6o7IUyt6KZaJM5V1LUO2QWf7vP2x3nLD3ZBw1Z0LsKmTjm7LY1l3S7h8OYJp8nDY/EPIf6mHvI8fMwUeGcf6BweBjTJP/J0/YC588Ya/E45gjNpexeH15JhXFPdQZi+B9h+6OBbPHY8b4Jvzj7W0Aopo/g1yATbSUm/M2Sr1h8m9rzGJHj55U+xPTXax4ln/WskKft0jU4dvakMERsycktnnbJ47G7c+v0foChhOMsmMwVjCmyT8J17zvAExpzeN3z60BAExuyUdq5MdFfjDynOFr40zW0VnrsuwjFhlrzmews7fM3xMCtOYzsbKPqFHfcP5hylLSMnqKDlzqDTzs2i44eg/8bekWLpOxGYguKU3GPru14qr37p/q2KFCIPsY9jcYGxiQ5k8IuYYQspgQ8jIh5GFCyHR/OyGE/JgQstzff5jwmQsJIW/5/y4c6AXUExccvQdO3X83LscwZy8QEGuS5S+C1d7f2au21mXyt6xgaUg5Vr81n4FlkdhoH5HHzjxoGj64YGZMbz2wSCLWl7e/eQauPmd/P9rHO4ZZ/iPBgVst6sX9ZkwxGG4Y6NP7HUrpQZTSQwD8BcDX/O2nA5jn/7sIwE8BgBAyAcCVAI4CcCSAKwkh9V2lowZgJCeuG8sDexQPNbOIbckpzJytHYIFLy7Gspsiu5UNIDL5j2MrjsVF+/SDcVihNuYHsS1vBmARRGr7xJ17pMFws8FYw4CeXkppp/C2GQEnngPgN9TDswDaCSHTAJwK4BFK6Q5K6U4AjwA4bSB9GAwwfX/auEa+jTk/VeF/LG9AtvxVZRbGC8swzhjfGNnP2pfXrG1v9D4XR8D9rUWTsQhOO2C3SFubOguYfdn9ePLNrQBGl+UfrOE7tP0wMBgsDPjpJYRcSwhZC+DDCCz/3QGsFQ5b52/TbVe1exEhZCEhZOHWrVsH2s0BgRU6E5Oa2Cin4goW525L0TuqRcfHNweRRnLSFBAMIM1SzP/UNm8WIjuJRfRXajj74OmhEg2AN4t4ZW0HAODh1zcnnnukol6ZuCbD12C4IfHpJYQ8SghZovh3DgBQSq+glM4EcBuAS2rVMUrpTZTSBZTSBZMn6zNSBwPbur2FysUlIBn7qwiWrZErW/5Z24qsUjU+tAB7VL9n1rttWfjAYTOw//Q2AMBUXyKK0/yrlX2yNsGLX30Pvn/uwdG2LBJZmH00kb9x9BqMNSQ+vZTSkymlByj+/Uk69DYAH/Bfrwcgehdn+Nt024c1WIx6SPbx2V9l0TGCV4VaytJPU8KqVWwAsQjwvXMPxjFzJwEI/ANxte+rzUglhGBCc05JhKptI6FcQ1oEVT1r1+YvL1yAU+YPzXqwBgZJGGi0zzzh7TkAlvqv7wPwH37Uz9EAdlFKNwJ4CMAphJDxvqP3FH/bsMa2rqjlT2Msfyb7qMiRkX97U+BQjYPNyd/7y2YhU3zZJ85irZbI4g5Xyfuj0VqOu6a4dRBUePd+U/Gfx+8JAGhrNFHVBsMLA/1FXkcI2QeAC2A1gE/52x8AcAaA5QB6AXwUACilOwgh1wB4wT/uakrpjgH2oe5gq3FNbVNo/kryV0f7AEBTzrvl45ty6OgtwyIE93zmnaG2Rdg8n8B7z2YhYuSRDtU6fGOzlUch0YdQxeVVcycW7DEeV5yxX2hRGwOD4YABkT+l9AOa7RTAxZp9NwO4eSDnHWzcedHR+NvSLaGCZcfv7fkhPnTkrMjxjTGWPyuNMGN8I1Zu68GkljwOnaWPdmWkyy1/fxYiO2RVqFr2iS1t4O1jSy2OVtR6jCOEcOvfwGA4wcxFU+CoPSfiqD0nhrbt3t6oLGIGBI5QlaTDEqT+7YiZOHm/qTg3IfGKRQyxQWDe1Ba8vrET09sD/8PMCY043i/T/KVT98F3HnpTe/44xB3O2po7pWVUkn+a2j7zprbg0Fnt+MqZ8werWwYGdYMh/zqAVfxUWf4sYmZySx5nHaQvrczALX9fQfrW+w/Ex4+dE7L8n770Xfz1xSfNxfMrd+DJZVurtmJjS1X4+yanmHGMRKSp6pnP2LjnM8cMTocMDOoMQ/51AA/PVETisOxY3WpXMgLN3/vblMvgoBnxpXmDhKVqZR89WEXS6e2NuOmCw7F259AvvVgPjHLPhoEBhyH/OoBxvsryH9eYxdauYuoKl7YQ6pkW7NCqnbQxh6/2pZ65U1pwyv676Q8coWCXPtr92gYGDIb86wAxMUvGzRcegceWbsb41OTvtVGNFc+LzlVJZHHnYPWI5k5pqa7REYbRGL5qYKDC6EnRHAb45PF7Yt/dWvl7leU/a2ITPlrFohosvr4a8u+37JPi8NmTmpIPGoHgmv/QdsPAYNBgyL+GuPyM/fDXzx3P39di2cL+WP6MwtJE+zz6heN5uGqaM8SVkxjJ4FE+hv0NxgiM7FMP+BlgtSh/YHMrPv1nqqlQOXdKK07cZzJuf35NrORx78XHoLeYvBjMSIcpwGYwVmDIv45Qaf7Vt8FCPauQfdjflLOFIJFMf8xoX/zblHQ2GGswsk8dwEo/1MTyl8o7pAE7Nm2SVzBGjV3mG7tXbjBWYci/jujPAuq6NvoT7ZP29P2NDhpVMPfAYIzBkH8dwFb5qo3lzxy+6T9T7WLkfHZRVc9GF07wazWdsPeUIe6JgcHgwGj+dURNLP9+hm16n61W8x+79H/4HuOx6roz+TrFBgajHcbyrwMYfQyV7MMiVtL6m4mRPDiMw9dgrMCQfx1RC0ua9Mcqr3K2YGSfACbD12CswJB/HVDtik9xsEj4b7rPVEdgLIvYEJ+BwdiBIf86ohZcGkg41cf5p4WJ9jEwGHsw5F8H1NJlaHGrPP1n2LFpZyCG/A0Mxh4M+dcBtIa6D7f8q3L4enBT9iPQ/A37GxiMFRjyH+5g2bpVVfX0jjWWf/+x1+Tmoe6CgUFdYeL86wDGubVwoPZnkRF2aNr5h9WPstGjGX/57LGYMb4x+UADgxEMQ/7DHNYAQj3Tyk9sVmGo38MBu48b6i4YGNQdRvapB3zOrQWZ9qfaJNPu01r+xuA3MBh7qAn5E0L+hxBCCSGT/PeEEPJjQshyQshiQshhwrEXEkLe8v9dWIvzD1fUglS55V9NqGeVug+bIJhBwMBg7GDAsg8hZCaAUwCsETafDmCe/+8oAD8FcBQhZAKAKwEsgEdNi8j/b+/eY+SqyzCOfx96FdvQlktpui1ttQbqrTYrKQGRANJSL6tJY5oYIWpsVEw0xtTWJgQ1/iGJoiZGglq5eAFFjQ3RYLE1/iMtt94QCqvUwKawaqVqTCrS1z/Ob3aHyey2szvn0jnPJ9n0nN+Z3fPsu7Nvz/nN2TPS9oj4x2RzVEl08WLPTu/ND6NnCaea48RI83f3N6uLbhz53wJs4pXHmQPAnZF5EJgjaQGwBtgREUdTw98BrO1ChkqZmd7qcGoX3sxlQlf70NnVPi27MrMamNSRv6QBYCgi9rUcNS4Enm1afy6NjTXe7mtvBDYCLF68eDIxC3fju1dw/lkzufqiyd8eePQ6/w4+Z+TI/9Q0zhB84G9WHydt/pIeAM5vs2kr8HmyKZ+ui4jbgNsA+vv7T6v77M45czqb1l7Yla/VaPoT+QvfU/0jr8bDfKmnWX2ctPlHxNXtxiW9EVgKNI76+4BHJV0MDAGLmh7el8aGgCtaxn9yJmZiAAAGd0lEQVQ3gdy10emtGtJndfQ5r5s/m/f39/HRty3rKJuZnb4mPCkdEQci4ryIWBIRS8imcFZFxPPAduC6dNXPauBYRBwB7geukTRX0lyys4b7J/9t9K7G0Xgnvb/TaZ8pZ4ib17+Z5fNnd5TNzE5fef2R16+AdcAg8B/gQwARcVTSl4CH0uO+GBFHc8rQEzqdwgH48KVL2fXkMGtePz+nVGZ2uuta809H/43lAG4Y43HbgG3d2m/v6/zKndeeN4s/bLkqpzxm1gv8F74V1+kUjpnZqXDzr7iR62+6+fZgZlZ7bv4V13jB94R7v5l1kZt/xY1e6unub2bd4+ZfcT7yN7M8uPmbmdWQm3/F+WofM8uDm3/Fjd6h0+3fzLrHzb/izpjQvX3MzMbn5l9xE7m9g5nZybj5V9xEbuxmZnYybv5V52kfM8uBm3/FzRh5S0i/0YqZdU9et3S2LvnY25dx/KWX+eAlF5Qdxcx6iJt/xZ05fSpb1l1Udgwz6zGe9jEzqyE3fzOzGnLzNzOrITd/M7MacvM3M6shN38zsxpy8zczqyE3fzOzGtLpcJ94SX8F/jKJL3EO8Lcuxekm5+qMc3Wmqrmgutl6LdcFEXFuuw2nRfOfLEkPR0R/2TlaOVdnnKszVc0F1c1Wp1ye9jEzqyE3fzOzGqpL87+t7ABjcK7OOFdnqpoLqputNrlqMedvZmavVJcjfzMza+Lmb2ZWQz3d/CWtlXRI0qCkzSVnOSzpgKS9kh5OY/Mk7ZD0dPp3bkFZtkkalnSwaaxtFmW+mWq4X9KqgnPdJGko1W2vpHVN27akXIckrckx1yJJuyT9UdLjkj6Vxkut2Ti5Sq2ZpJmS9kjal3J9IY0vlbQ77f8eSdPT+Iy0Ppi2Lyk41+2Snmmq18o0XthzP+1viqTHJN2X1vOtV0T05AcwBfgTsAyYDuwDVpSY5zBwTsvYzcDmtLwZ+EpBWS4HVgEHT5YFWAf8muyt5FcDuwvOdRPw2TaPXZF+pjOApelnPSWnXAuAVWl5NvBU2n+pNRsnV6k1S9/3rLQ8Ddid6vATYEMavxX4eFr+BHBrWt4A3JNTvcbKdTuwvs3jC3vup/19BvgRcF9az7VevXzkfzEwGBF/joj/AncDAyVnajUA3JGW7wDeW8ROI+L3wNFTzDIA3BmZB4E5khYUmGssA8DdEXE8Ip4BBsl+5nnkOhIRj6blfwFPAAspuWbj5BpLITVL3/e/0+q09BHAlcC9aby1Xo063gtcJUkF5hpLYc99SX3AO4HvpnWRc716ufkvBJ5tWn+O8X8x8hbAbyQ9ImljGpsfEUfS8vPA/HKijZulCnX8ZDrt3tY0NVZKrnSK/Rayo8bK1KwlF5RcszSFsRcYBnaQnWW8GBH/a7PvkVxp+zHg7CJyRUSjXl9O9bpF0ozWXG0yd9vXgU3AibR+NjnXq5ebf9VcFhGrgGuBGyRd3rwxsnO4Slx3W6UswLeB1wArgSPAV8sKImkW8DPg0xHxz+ZtZdasTa7SaxYRL0fESqCP7OziwqIztNOaS9IbgC1k+d4KzAM+V2QmSe8ChiPikSL328vNfwhY1LTel8ZKERFD6d9h4BdkvxAvNE4j07/DZeUbJ0updYyIF9Iv7AngO4xOUxSaS9I0sgb7w4j4eRouvWbtclWlZinLi8Au4BKyaZOpbfY9kittPwv4e0G51qbps4iI48D3Kb5elwLvkXSYbHr6SuAb5FyvXm7+DwHL0yvm08leGNleRhBJr5Y0u7EMXAMcTHmuTw+7HvhlGfmSsbJsB65LVz6sBo41TXXkrmWO9X1kdWvk2pCufFgKLAf25JRBwPeAJyLia02bSq3ZWLnKrpmkcyXNScuvAt5B9nrELmB9elhrvRp1XA/sTGdSReR6suk/cJHNqzfXK/efY0RsiYi+iFhC1qd2RsQHyLte3Xy1umofZK/WP0U237i1xBzLyK6y2Ac83shCNk/3W+Bp4AFgXkF5fkw2HfAS2VziR8bKQnalw7dSDQ8A/QXnuivtd3960i9oevzWlOsQcG2OuS4jm9LZD+xNH+vKrtk4uUqtGfAm4LG0/4PAjU2/B3vIXmj+KTAjjc9M64Np+7KCc+1M9ToI/IDRK4IKe+43ZbyC0at9cq2Xb+9gZlZDvTztY2ZmY3DzNzOrITd/M7MacvM3M6shN38zsxpy8zczqyE3fzOzGvo/oCXxo3Z1n/cAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mV5jj4dThz0Y" + }, + "source": [ + "In addition, `avg_final_reward` represents average final rewards of episodes. To be specific, final rewards is the last reward received in one episode, indicating whether the craft lands successfully or not.\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "txDZ5vlGWz5w", + "outputId": "c7c5e9ca-6329-4ee2-f3d6-a1ba46b5aea2" + }, + "source": [ + "plt.plot(avg_final_rewards)\n", + "plt.title(\"Final Rewards\")\n", + "plt.show()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEICAYAAAC3Y/QeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO29e7wkVXUv/l3Vfc4MM+MwPIbnMDwHFVABJ6jRgA9UMA+MUX/k3sRcrwlqNNFcE6PxJj/NveZnkqsxRhN/JOIjapT4gkQTBUUQlccAIzA8h+cwDDADzHvmnNNd6/5RtavW3rV3dXV31TmnT6/v5zOf0121a9eq6qm1V33Xd69NzAyFQqFQjBeiuTZAoVAoFLMPdf4KhUIxhlDnr1AoFGMIdf4KhUIxhlDnr1AoFGMIdf4KhUIxhlDnrxhJENFuIjqhhn4+SERfrMOmuQIR/ZCIfnuu7VCMFtT5K+Y1iOhBItqXOnvz7yhmXsbM9zd87pcSUZyecxcR3U1Eb27ynArFbEGdv2IU8Mupszf/Hp3Fcz/KzMsALAfwBwD+kYieOYvnz0AJ9JlV1AL9j6QYSRARE9FJ6efPEdGniOjbaYR+PRGdKNr+LRFtIqKdRHQTEf1Cv+fjBN8B8BSA56b9RkT0PiK6j4ieJKJLiejgdN/nieg96eejU3vfkX4/kYieSo8/iIj+nYi2EtHT6edVwvYfEtGHiejHAPYCOIGIXklEdxHRDiL6JAAS7U8ioqvTfduI6KsD3F7FGECdv2Kh4EIAHwJwEICNAD4s9t0I4HQABwP4MoB/JaLF/XSeOupfAXBo2j8A/B6A1wI4B8BRAJ4G8Kl039UAXpp+PgfA/QDOFt9/xMwxkmfwswCOBbAawD4An3RO/5sALgLwDAA7AHwDwP9MbbkPwItF2/8F4HvpfVgF4O/6uU7F+ECdv2IU8C0i2p7++1agzTeZ+QZm7gD4EhJnDwBg5i8y85PM3GHmjwJYBKAqdXMUEW1H4pS/CeB/MPMt6b63AfgAMz/CzFMAPgjg9UTURuL8X5LSNGcD+CvkTvqcdD9Su77OzHuZeReSQescx4bPMfOG9NrOB7CBmb/GzDMAPg7gMdF2BslAchQz72fmaytep2LMoM5fMQp4LTOvSP+9NtBGOsC9AJaZL0T0h0R0Z0qFbAdwIJKouQoeZeYVSDj/TwB4udh3LIBvmoEJwJ0AugAOZ+b7AOxBMgj9AoB/B/Bomi/InD8RLSGi/5+IHiKinQCuAbCCiFriPJvE56Pkd04qM8r970VCA91ARBuI6L9XvE7FmEGdv2JBI+X33wvgjQAOSh35DgievArSyP6PATyHiMwAtAnA+WJgWsHMi5l5c7r/agCvBzCZbrsawG8hoWTWp23eg+Qt5AXMvBw5NSTtk6V3twA4Rlwfye/M/Bgz/w4zHwXgrQD+3uRGFAoJdf6KhY5nAOgA2AqgTUR/hiSK7xvMPA3gowD+LN30aQAfJqJjAYCIVhLRBeKQqwG8E0k0DwA/TL9fy8xdYd8+ANvTZPH/28OMbwM4lYhel9JLvw/gCLOTiN4gEsZPIxk44n6vVbHwoc5fsdDxXQD/CeAeAA8B2A+bJukXlwBYTUS/DOBvAVwO4HtEtAvAdQBeINpejcS5G+d/LYAl4juQcPYHANiWHv+fZSdn5m0A3gDgIwCeBLAGwI9Fk58DcD0R7U5te1fT8yEUownSxVwUCoVi/KCRv0KhUIwh1PkrFArFGEKdv0KhUIwh1PkrFArFGKI91wZUwaGHHsrHHXfcXJuhUCgUI4WbbrppGzOv9O0bCed/3HHHYd26dXNthkKhUIwUiOih0D6lfRQKhWIMoc5foVAoxhDq/BUKhWIMoc5foVAoxhDq/BUKhWIM0ajzJ6JjiOgqIrojrS3+rnT7wUR0BRHdm/49qEk7FAqFQmGj6ci/A+A9zHwKgBcCeAcRnQLgfQC+z8xrAHw//a5QKBSKWUKjzp+ZtzDzzennXUhWOjoawAUAPp82+zySdVAVCoUHl63fjF37ZwrbuzHj0hs3odMdvFz/tt1T+M/btwxj3rzF5u37cNVdT/Rsd//W3fjJfduy7zc99BTu3LLT2/bux3Zh+97p7Pu3btmMPVOd4Y2dA8wa509ExwE4A8D1SJa5M//jHgNwuKf9RUS0jojWbd26dbbMVCjmFe7cshPv+sp6vPdrtxb2ffmGh/Her9+Kz/80OI+nJ9782Rvxti/ejB37ioPLqOP8j1+DN3/uxp7tXv7Rq/Ff/vH67Puv/cNPcf7f/sjb9jc+cz3+6UcPAABuefhpvPur6/Gn37q9HoNnGbPi/IloGYCvA3g3M1tDaroGaWFRAWa+mJnXMvPalSu9s5MVigWPfTPJgl9bduwv7Nu+J4lAn94zXdhXFZue3gsgeYtYaNi5v/6IfO9UB3umk373TCW/zeO7ir/NKKBx509EE0gc/5eY+Rvp5seJ6Mh0/5EAer+bKRRjjDLXzKV7y9HXQsYKxAyY9a+Gue/zAU2rfQjAZwDcycwfE7suR7KQNdK/lzVph0IxqihzzqSee9bBYLirH9KIDqFNF3Z7MYDfBHAbEa1Pt/0JkvVHLyWityBZV/WNDduhUIw2SpZb1ZVYZw8xJ/+A0b/vjTp/Zr4W4eDlFU2eW6FYCKCS8L5sn6IhcJHuGdWfQWf4KhQjgHLOv4b+Rz2MnSXEzHnkP7emDA11/grFPMaIBpULFozRp3sM1PkrFCOAphyOUkf9IeZiwndUoc5foZjHqOKbF4gvagx1OmuWUs8Rv/Hq/BWKEYBPU15n0D7abqwcdflo4+xjV+o5om9P6vwVihFHHZONRjyILUVdl8ZOonfUb5k6f4ViHqNsAlGdk4tGfbZqGeqiZ+JQ5F9L77MPdf4KxQig1H8N4duM41rIkX9dZYvY/TDi90ydv0Ixj1FGJ9fK+Y+4IytDXW81wch/REN/df4KxQigzDnXMslr1MPYEtSX8E3/mu8jfs/U+SsUI4o6A86FHPnXBXOPXBppRAN/df4KxSiglPKvwXMvZN9fW+Sf3iVzv0d9wFTnr1DMYzTN+Zs+4gW4mIuBy9EP3k/y1+1Odf4KhaIxlEX3ox6BNo36dP5pxA+N/BUKRcMoczC16vxH3JGVoT6df/o3trePZtyvzl+hGHmo2qcctV2Zs3zjqN8xdf4KxTxGaeSvOv9KqOvacp2/vX1EKX91/grFfEYVfrkO57aAfX9tF5fp+7Wqp0KhaBrN+5ckbK1LETMfUZ/aJ1TXYTRDf3X+CsU8RpXZpFrVsxx1V/XUZRwVCkXjKKMW6tWXj7orC6Mueiaf3KW1fRQKRcNweWZvmzo4/4Xr+ysPa70GCbM3NNlr1KDOX6GYxyjX+dd4nhr7mm+o6qR7tYuzSV42RjTwV+evUMxvNKspz8o7jHoYW4KqtE+vVkWVz2jfszlz/kR0HhHdTUQbieh9c2WHQjGfoTr/4VEX7ZNF/qrzHxxE1ALwKQDnAzgFwK8T0SlzYYtCMZ+Rc/7NeucF7fwrXluv2na52mdh3Ky5ivzPArCRme9n5mkAXwFwwRzZolDMW1TxM/WUdF4YDs2HqtfWq11O+9h/68JXbngYT+zcb237wk8fxMXX3FfviVLMlfM/GsAm8f2RdFsGIrqIiNYR0bqtW7fOqnEKxXwBB5KMgC7mUhX9JHxLq6cisIxjDb/EYzv2433fuA2//YV11vYr7ngc37ntsaH792HeJnyZ+WJmXsvMa1euXDnX5igUc4IqfmsB++1a0M/9KRso3Mlddd73mW5SKvTJ3dOFfVFDOYW5cv6bARwjvq9KtykUCoEyYUkdk7xMDwuFx/ah6kI1zOX3wZ3kZdo2mfCNmRtbLGaunP+NANYQ0fFENAngQgCXz5EtCsW8RRW+Wid51QPucbfdyV113jPTl+vnmZubR9BuqN9SMHOHiN4J4LsAWgAuYeYNc2GLQjGvUVJHplapZ31dzTv0o/YpfwOyI/46I38z7Hidf0Pef06cPwAw83cAfGeuzq9QjAKqcf51FHZbuO6/stqHGcxhT9tkQbcs8nfifAbXumKbxLxN+CoUivLa8VreoRoqq316tHWremaRf42/hC/yb4r3UeevUMxjNM35G2ezkCP/qsls5oqlswMzfYdBqCvGwlP7KBSKCuAyqqFGMngB+/7qbzVcPsvXLNweO28AdUTmnL1FFLc3RfvMGeevUCh6Y7Z0/gvY9/dB+3A57ZMV2bMln3UgG0ecAX1BJnwVCkVvcBnFUKfzWcjev+LQFjNAVTj/2P5eh28O9cVozvkr7aNQzGP0Fh7WpfNfuN6/enkHLr3hLgVXbz0kv/dvkvZR569QzGdkDqfoaGYj4bgQUPXaGOXJ4byks9H5J9vrnIGrkb9CoQDQQ31Sw6IiJqpc0OUd+lL7lOwX7eTfOhDqK+a612rOoZy/QjGPUeZoanXXC9f395XwLVX7ODN7c53/8Ai+RTAvrPIOCoWiGqpMOqqF8x++i3mLvko6V+hn+PetIkIDSZO0jzp/hWIeoyypW2u6cQF7/+rlHarRbC73X4dzDtUJarKwm3L+CsU8RuniIjXONF3QK3n1o/Mv3W9/aKSqp6+2j3L+CsX4oWmXbPxKxZL3I4l+aJ+45I6bdQFMi3o5f/9547i58g7q/BWKeYxKnL9W9SxF9TV8UTrauk6/zluWJ3x959TIX6EYQ5TQPjW+Fyxc19/fJK+ypq7Tz3n64Z1zKPJnZk34KhTjiLKSzrXqzRew9688yatXQ8fp1wkuGUhU6qlQjCEqJSBrOc/C9f5VKa1ezdxlHBvR+Xts0shfoRhj+PxSHQGo8SsLmPLvo7xDeUu3mmetnH8g485gRKr2USjGD+UzfG31ySAwNMPCVvvUHPnD/ltH6B+6/7FG/grFeMKtIW/tq1VnvnC9f3/LOFaf5FXnMo7hBdy1qqdCMZao4ri0vEM5qid8e0zyYv/fOpBN8vJJPTXyVyjGD6XlHXQxl0qoem1xD6G/GRryxG8D5R0KBf21vINCMZYopyHSv7qQYymqD5LlyziaFbyQJX6HscrpuyTybyrhq85foSjBz/9/38df/uddc22GX+1TY/8LOeFb9dqYy9vmM3ztv42Wd2hwkldjzp+I/pqI7iKiW4nom0S0Qux7PxFtJKK7iejVTdmgUAyLR3fsxz/88L45O3+V8g51jAILmvbpo7xDWVu3mmetM6xDJZ1HlPa5AsBpzPxcAPcAeD8AENEpAC4EcCqA8wD8PRG1GrRDoRhZcAnFUG95hwXt/as1Y0nt+PcDxci/DmTndcL8Jqt6Nub8mfl7zNxJv14HYFX6+QIAX2HmKWZ+AMBGAGc1ZYdCYcDM+PTV9+HJ3VONn+uJXfvx+Z88OFQf6zdtx7dv3VLYvvGJXbj0xk2ZM7rq7ifwx1+7FRse3VG572vu2Ypr7tmafQ9F/v95+2NY9+BTfdltcO292/DxK+8Z6NgQ9s908aF/24BPfP9efP2mR/Cje7fiH6+5vzw34nzfsmMfLrn2AU87u6jzJ75/L2a6+WjATuRvbtptm3fgsvWbB7wiYMOjO/CNWx4BMLuR/2ypff47gK+mn49GMhgYPJJus0BEFwG4CABWr17dtH2KMcCWHfvxkf+4CwctmcD/83PN/p/63S/ejHUPPY2zT16J4w9dOlAfr/3Uj8W33Cl97abNuOTHD+Bt55wIAHh67wy+um4Tlh/QxqlHHVip7zddcgMA4OgVBzi92/jo9+7GmsOXYe1xB/dt/2985noAwLvPPbnvY0PY8OhOfPbHDwIAJloEIsJ0J8YFpx+Fw5Yv9h7jjgtv+dw63LFlJ8477QgclV4/kETfsu3HrrgHL1lzKM5cfVDSj9OfifzvemwXPvztO3HB6QU3Vgn/csPD+M5tjwHwL+YyL6WeRHQlgCM8uz7AzJelbT4AoAPgS/30zcwXA7gYANauXbuA30kVs4VubEv1msRTe6fTc5bwCAMiZk6iT8erdYa4sFDk3GUupUJmG11xjcn1svjsh0tp7dg3U+jLtHNvg7wvGefv6dftqx/0OnZelndg5nPL9hPRfwPwSwBewfld3AzgGNFsVbpNoZgVzG5ys54HV9rMzN71ZkP1Ycpg/ErwnvD8ygdIRyltLnOg7q7QNSdqHy5scz/nM3zF+Yf4TyVtd/+3xNzcAu5Nqn3OA/BeAL/CzHvFrssBXEhEi4joeABrANzQlB0KhYE7Lb9RNHiKmE30b28fxgGFHHzMPPSbUp2T0cokkVXPnzl/X8mMQr/Fc/hm+A4X+UvbnIQvj2Ztn08CWATgivSCrmPmtzHzBiK6FMAdSOigdzBzt0E7FAoAclJU1fbDO626HlxpiYn6XefVHYKeCV0ql+yripkuY7Jdz40IOdky5+vuMbNoq0X+xX5dyScw2FuX219im2MTmqvt05jzZ+aTSvZ9GMCHmzq3QuGD76EtwzBOr8l3CxP1u/YN44BC15rlF4bATDfGZLsekiH0dlP6NheifTx9FO6pL/L3dFsb7eNJ+I7cJC+FYr7BVWv0wjAPtEFdz60/ArW/D0f7hM47/EA2M8wriYPQAFd2CvcNKV/DwE34+rYVcwxZdU9hyzBJ8a4V+bs6/3qWifRBnb9ibOCW5O2FYXjcJkskh95ghkn4hteQHT5HMl2j8x+E9nEds3Gm7hG+qp7y0t2VvOqK/Mt+tybX8FXnrxgb9FuKt47EcF1Rm8v5A0WHN9SbSojz91Ah/WKmO8cJX+d7WeTvOmJXZSXPJc9Zm9TTR/sM3HM51Pkrxgb5tPxZiPwHPrI3fFJDYFh7Q051+EFwplNn5B/aXl3tY7yp25dXPiuOdSP+unIu5Qlf5fwViqHRr159PlW6tCLQ9K/rlAdx0r3ehurQ+NfJ+YfebsreekKRf3HA4KLax/ps6Lb0u9N20Dev8oSvruSlUAwNw/1WdZJ1qGea4P5DuYtBIn9zTOjIOiL/Ojn/0G9Szpvb3/N1i93BE4UbIdvIU/jyA4O+eUlWzJ/wHajbnlDnrxgblFXI9KEOtU9drp899EOB8x/Ax5qovCzhO584/0ESvu6vEIr8k+R2+FB3tm8db15A74FLF3NRKIZEPj2/WvuhIv9soKk/8g9x/oM4n+mUjw/SPlykQvpFZ45pn1B5B/eYJJoPO3T3s3vKgSP/Etqnydno6vwVY4M8YVcx4VtH5F/Ts2txz2YQK0T+/Z/MROWhIxnDX0OdtE9oMK1Sh9/AUCsFZQ+KA4Uv1xJqO6jW39L5ezK+SvsoFEPCrc3Su/3g5+r3LaMfhFRLg0SJWTK2REI5qPNvRZSeo07ax7+9XOrp0D6Z2qdI+xTmTlh0m/sWUE/CVw5CXs5fE74KxXAwj1hVKmYY2ic/Z/2hv+mzDtqn0yPhm8gfB7uGVupla5V6DqL2Cewq0D6eK/W9cZnPBannoGqfkshfJ3kpFDWg38h/uBm+6TmbqIUfiPzrsNfFMJF/lHqX2SjvUJo0db5nah+fzr+QBxDncJLuhTevQXX+PWyP1PkrFMOhXyqmHrVPPZG/7CVUmnqYgSYYtXqcXFWYyH+uyzsUSjqbYzyO3r1Un8oKgHfhl4F1/qXJ6hFcw1ehmG/I1mCt6JBroX0a4PxNl3WWdyiN/AfsM2qA8x+snr/9Patn5FnJq5DwDXxO5j/YbQd984p71fMfqNfeUOevGBuYR7Oy1HOohG9/FFPV/oBigTGDJspR+FQtVZEnfGcj8g8f00TC1ysLHfAye5V3aMr7q/NXjA3ikNcMYBhnmp2zwRm+rn1Vz+VLeAcllEPM8jKTk+a8vIMb+afetJjw7SH1tPh/z+A7KO1TovMHq9pHoRga/Uf+w0fSdbl+n+pk0IRvP2Oad9ZrRRjnP12j2qee8g7+Y9gj3wxN8oLnLWHw8g5lkT9rwlehGBb5SkyzF/k3UtsnIPWsaq+vXWl5h0Glnql36dRwHw0GquoZ6suT8C0z1U261yG1BcoHrph1kpdCMTyyiLla8zoSqI1M8srmZQ1G+/jahcU+PDCX3U61nnOt83evl7LtdjufgicU+RffEeqJ/F1oVU+Fogb0SfnXFLXX4/1dmSFQdBrDRP7BhC8PfwVzrfN3LyDX+fsi/+I23+ck8q+H9pGDq9sDQyN/hWJo9FtsbRifFaJm6kBe3sG/vRd8kWb5JK8BnVp63HSd5R0GSfj2o/ZxjnXLOMu2ZeWf+4G0ozjPQKWeCsXQMA9W1Ue0Hs5/6C6SflB0EIMWdvNFySFenzH4NRh75jryL6p9EvjUPsUZvn7H7Kt2Wgft4+1BJ3kpFMMhmxlbWRUzvOduUuo5qPPx0j4hzt9DhVSFOW52dP7VE75h2qfI+bvyTtlnWX6gH8RW5F/8rGofhWJIsPO3F+ZV5O84HmBwnb+f9glvG/QSmoj8w7RP+JhQwtfty7dqmfXGJT77Of+wDWWwIn/PYDOyCV8ieg8RMREdmn4nIvoEEW0koluJ6MymbVAogHDEHMJQOv8suVx/5B8qUFed9ilu85kZKh1dFcae6U5996BJnT+8ah//Z19+oI7FXGx6jy1760ajzp+IjgHwKgAPi83nA1iT/rsIwD80aYNCYZA75Grt4xoevkHdXoF7tvYlf4uR6xCRf5kNA16E8WlzPsPX+Z7N8O1T7SO/+BLh9dA+4nSZvc2g6cj/bwC8F/b9vwDAFzjBdQBWENGRDduhUAipZ1VuPPk7yMNnzjA4X162r/gG04pouIQvA5u378PHvne3eEOyz3f75h24+eGn8aF/24B9092e5xmW9rl98w586fqHsH+mi098/15Md+KBJnmFMr5ddjh2FAeK793xGL5/5+MAPJF/n29eV939BL51y+ai7Q7t8+1bt+B7Gx7L+m8q8m830y1ARBcA2MzMP3Mq1R0NYJP4/ki6bYtz/EVI3gywevXqpsxUjBH65bDNwzzMAtrfvHkzli+ewPOOWdHXcQVH4qNlRJuJVnXn79f5M9755Ztxy8Pbcf5zjsSzj1yeS2PTNr/0d9dm7V95yuH4+RMPLT9PlvAdbAA059u2axp/c+U9eMbidj31/MUxro7fDQx+ePdW/PDurXjwI79YUFwVOP8eA/2bP3sjAOC1Zxzt2C5tZbzjyzcDAO7+3+cl9jbk/Ydy/kR0JYAjPLs+AOBPkFA+A4GZLwZwMQCsXbu2fuJUMXbol8MehvYxp/jGLZsx0Yr6dv5lNvoWcJ9oRUPp/GPOI/SOWde35H51Kjj03CEP9/hOdZK3jD1TnXoKu4kF3OWuXgvX2Gqfojh20BLg0nb37ULaWzeGcv7MfK5vOxE9B8DxAEzUvwrAzUR0FoDNAI4RzVel2xSKhuFPlIaQOf8hWddBqJ8CH+3R+cs2k60IeytQMUDASTGjlZZj6KShaFmOpAqV0/UMUoOgnWodOzEPFPkX1T455+/SPlXZozrr+XftUaVwvpFS+zDzbcx8GDMfx8zHIaF2zmTmxwBcDuBNqernhQB2MPOWsv4UijoQmhkbQvZQDvTs+aO5qvDJEPPPRc5/ohVVrkUUSvgaJ2uuu2zZy17OX2rmh53rYAalblyUVxoMEvnHsR29+2r0w9lvta0h4etbUMb9PC8j/wHxHQCvAbARwF4Ab54DGxRjiPzZ7I/2GXaSzSByT3+5YfM5bSOdf5sq0w6hSV4tEWEDYl6Ex/5eJRvkOYaO/Fu5XaHTli/mYsOifZyd1at6+hK+4WND8FUWdT83pfaZFeefRv/mMwN4x2ycV6GQyGf4VmtvHuZBEr42RTA87ePjgi3OP6oe+Xt1/uBszV038vc5xF6VOrtOlDwMIjEjN0j7lEb+YaqoQKeV5VqscxffEgapAluk94qfR1Lnr1DMJ+Tpx6pO0nD+w6EO2geQaqUi7dNukVetUr1vO8I22+T5JHrRPnKAGZb2Mc6vE/NA5R1cGNvcyN+3hq+ETREVB9FBEr7uvfEtGzmM2qwM6vwVY4N+19XN1T4DRP6efvpB2SzcLPK3pJ45L96z74Dap5Vx/ibhWxL593D+loJlyDleWTmGmIPRdRV1lPvdvYQ4Lh887dr+xUFxkISvb6KZPEeTUOevGBv46JIyGEczbNw1SOBbNgs3JPV0t4UQquqZJ3zTbc5gI9Ef519X5B/XUt5B5kzsyL88G+S+JRTzBcMP8uz50pTOX52/YmzQ7zKOGe0zkM5/OOfnc2a5/QlcqWfVc3kjVE/kn/c1CO3jcOlDQEoz6yjvkEf+Luffg/Zx3mbqcP4Fuy3paT3BRwjq/BVjg7JI1odu5vznXucPFN9cpDOaaPvr1Xj7Dko9jc7fHmSGTfjWFvl3E86/5ZFflap9Ak66687wRTntY3H+voTvIGqfsoRvFvn3328VqPNXjA1y3Xo1Z2SYjUGknjbn3//x/lm49ugl22Scf4Vr8+cTcqdqZu+W3a9+Iv+61jQwOn+f8y8v7+By/rldLs0SMjV25hgweyZ5DfKGVyb1TP9qwlehGBJlkay3/RAJX18//aB8xqr9FxCc/4CRf8z5JK/MsZfkSHpy/k5ydBgYB2nUPhO+yL9U6ul+l5G/TbOEKMGZOO5ZB2gQtU/Zmgx1VJUtgzp/xdhg0MJuA1X1FCepO/LPpJ4ezr+S2qfHJK+ZLPI3+/qP/O11aYfz/saebsyIY/Qf+RfUPuavW9sn/FvNdO18QOzJD5SuJhbKVZSofTLaJ9jrcFDnrxgbSJVHFXRrivxr5/zjYr8TqUa/CvUQquppdP7GsecL3hf76F3eIf88bORvaKhOHKPLnL3lSPQT+Qc5/xLaJ8lxyLeEcCLZe3zgTam4cpj83Czpr85fMTbot8hkfWqf/o/38fJlap+c9undd2iSV8uhfbLI39NHP5H/sJy/KTTXjcsSvmWcvw3TtGv783SSV4D26caFiWtuNF9mQ+h+FSP/Iumvkb9CMSR8M2PLkJd3GPK8dUgAIWYoZ4nYfN9Eu5+Er7+NUftMd91JXsX2vZZmrJPzn8ki/8Q5+yL/8vIO7vf8unzVUn2Y7saFtm77snsvnX9VGbDZo5y/QjEkcg67WvtB1BsGNpdcE+3jTL6S9tKwVxoAACAASURBVE0OmfBl5kxVMtOxB5dBaJ+4Rs6/0x028nc5/wDtg/D9czn/ZE6A02+JDdPifklbC7OMHUURoGofhWJ4lESy/uZhzrsfDFLewGej++bi5fyrOH/vBLI8wsw4/5IcSV/lHYamfdLIP9X5m9yEdb4Sc9zLzWifgtQzPP1vphsXBopC5F9ig+T8y+6NnStJacdwt0NBnb9ibFDGYftgnORArsvzEPeDkIOWXctBpS+df9pmQjhRyXfnnH/4+nstzWjsb0c0mNpJHGTs6abRdtun9im77oLaJx08Halnovbx9zPdiZ0ZvsXIv5T2EZPi5O+mVT0VillAHslX5Pz7bB8+b//H+OWYZZF/H7RP6nwMx29sNN0Zx+7mGCSqVvVst2igwU/23xFSz27Mlt0GZYlT9+wZbRZ7FnMJmDrTjb2OWaLs3svrKYv8fVU9R2olL4ViPqJfzt88zIM476E5f88xrlrJ4vwHSPi2rcgf4ch/AM7f2DERRQPdP9n/TDqSJAlfeGmf0pmyBdonHzyLVE4Z52877b4if0n7xP7PQKAmkkb+CsVwyOiSPtU+w3LWtSV8Hc7fUvv0w/lntI8d+bvOfyjOXwwwg7w5SWeZR/5xGvn3TvjaUbo/ui4WdgtTfC7n7ysFUTXyLyt94aPTNOGrUAyJfmf4lnHeVc+V9NP/8d6Er8P5S/Sl8xd8fN53rmZxpZ6+8/Uq72Dsb7eiga7fon1M5N8djPYpJHyzxVzg6PxLOP9u7ET+xbZVdf7yDcEdQ33zIzThq1AMiTySrda+jPbo77yDRP6+fpK/Pgc1WMLX5fxN5O9y/sU+elb1jA3tMxjnPy36t8o7sF/tUzpT1jl9ljvph/PvuJx/URlUdu+nA5G/O2B0PLSPJnwViiHRd1XP2KZZ+oHN+fd9eEDtEx6M+qntIykZAwZnEbFx7L7ko0FVnX8rXV6yX9gJ31znH6rqWVojJ1DV01feoay2j/s20R/tIxx+ScK3K17dMspfnb9CMRzKIlkf8hLKw513oMVcfLSP+evpztTz72cxF5v2CXP+Sb92H/0kfAdT+wjO3+j8Y0Y3RqC2j/2dXT5HIOP82eH8y6p6ujp/9pR3qCj1LEv42pG/oX2U81cohkK/K3kNFfmXOM5+zi2Rq49KaJ8qVT19tI+wc7pbJfIvP498uxhW6mk+d1LevUpVT1fFY7U19Fkfhd1czt/3llA+ySug8y/JG2jkr1DUhIwzrzjjtt9JYda5EHac1c4dPsbn3wfS+bfsyJ/7iPyne9E+JuEbDZbw9ZVDyOr5e2f4hk9SvP8y8re3hqWe7gzf4ltC5fIOkvapEvmr2kehGA7cZ+QfDxH5W/3UJPUse3OZHCDha0/ykjN8i7mFoiSxV8I3+TsxqNRT0CTGISb1/Bktj9qnvEyC2zb968zwLeX8O7G1L46LQUQdOn97HYTk70iqfYjo94joLiLaQER/Jba/n4g2EtHdRPTqJm1QKAz6VfvkM3wHP1c/57PO7Z3hG+4vX3x9QM5f9OvW8wfsiBSorvYZXOopzt3N30i6XFXnX3SiBhbn7wxwQbWPowv1Naus8y9N+M4e7dNupluAiF4G4AIAz2PmKSI6LN1+CoALAZwK4CgAVxLRyczcbcoWhQIozpDthay2z3CB/9A5g2xbyT7j/Kucy7RpFXT+yXYjs5Rduc61F+efv13Ux/mbWbZezr/kHCF6pmzNBBfTKe1DlCfH+5vhW62qp2V3Fvk34/0bc/4A3g7gI8w8BQDM/ES6/QIAX0m3P0BEGwGcBeCnDdqiUOQrUwW8/4ZHd+A/bnsM73nVySCiUprlR/duxRevewjveNlJeOTpfYgIOO+0I/GTjduwe6pjn1cc/uTuKXz0intw6lHLcfCSSZz/nCO9tvgcySXXPoBHnt6LbbunCvvaWeSfOO8/+trP8NSeabzg+IPxzpevsc7/8SvvTY4R3Pm31j+Kw5cvApA4qi079uFPL7s9t0c4LCLjDLnAR199z1Y8sHU3DlmW9DXRiqyI+EP/tgFbtu/HX7zuOVgy2cKf//sdeM8rT8YXr3sYt23ejlc8+3Dsmergsz9+MDvGvHXMdGMQIVDYDXh853588gcb8We/fIp1zz/74wfx+uevwqlHHQgg/z3u2LITP7p3W9bur797N17xrMMKfQPJ20fMjBYROsz47obHsGXHfquNuUf/um4TGMAtD2/H6ccciEXtljVv4aEn9+DSdZvwp794SvkKZDCcf7DJUGjS+Z8M4BeI6MMA9gP4Q2a+EcDRAK4T7R5Jt1kgoosAXAQAq1evbtBMxbgg43oDz9sVdzyOT161Eb//ijWYbFMh8l+/aTsOXjKJ1YcswQcv34D7tu7B6ccchO/f+TiiiHDeaUfiH390P7btnnbOm5/wL75zF75+8yMAgHNOXhl0/h1PSPjP1z1U2HbARAuvf/4qLF2UPMrdmPHI03tx2fpHQQTc+/huy/nftnkHAOC5qw4slA14fOdU1sePNz6JWx/Zke2Tzv8VzzoMV975hLe88jdvfgQ3Pvg03nveMwGY8g55H8ap/8YLj8WKJRP48vUP4+w1K/E3V94DAPjBXU+AiKzzmXvRiRkR+Wv7dGPGT+7bhn++7iH8txcfh8OXL7b2v/HTP8WGPz8PQPJ7HLpsEtt2T+M/bt9itfv+XU/Ah06ccP6tiNCJGZetfxQA8IzFbeza38n6BYAv/PQh3LFlJ7ox46s3AmeuPgivPOXwrK+3f+lmMAO/duaqbGD89bNW47L1m7F3OidAzH0bdjGhEIbi/InoSiK63fPvAiQDy8EAXgjgjwBcSn2krZn5YmZey8xrV65cOYyZCkWCjMP3e3/zIMpyAkD+UL/2Uz/G2X99FQBbBjrTjS0H1XFmjkrnH+J+XUz14NQNTj9mBf7Xa0/LqBBJ3yybbBciS7Pvzy84Lagi6abXZG1Lr/dPXvMsnLH6oKydC0PN5HmFXOfvljjwSWljzs/15hcfl/Up94fKO2S1mOKiBt9K1jLwujNX4bBnLKqUIzH9x8zWZLo1hy3Dq089Iu83ztvm15Zeq5NYBpKaPabd2845ARecbsfA+X2Zh7QPM58b2kdEbwfwDU5+hRuIKAZwKIDNAI4RTVel2xSKRmEepdDzbh7QmQ4Dk8CMifxL+oxjxnSX0c7eKriQ+AvpustrwVRzSsZ/Zwlfzp3gRDsq2GL2tYiCEWUcF9U8xu5IHOfjzKe7seX8Jlp5PX+3xIHpM3QfjKPtOCcK1fM311pYpAV2fiNmBlEexVeBGawm2hEwlfdJThvTv2VbXPw/YY6X99UdizPOfz5G/j3wLQAvAwAiOhnAJIBtAC4HcCERLSKi4wGsAXBDg3YoFAB6F2ozvsk4KRPNl+UrTZScLThinJo4xl0EJD9fuOPpipG/cQyGwnEdb2gSURSFq0V2Yy6cP1/MnmDmhvkj/9hSzbREwted5dpLSmvmLnScgbAVoH3kYOJ2KccLZjOIUeXIP/bMMXDvXzb4uAOuGJCt64hEGYyomNZturBbk5z/JQAuIaLbAUwD+K30LWADEV0K4A4AHQDvUKWPYjZgHEKQ9nHoCel0QjRCHHPq8FK1TVyU/MmvvqqNPvTS0RsYlyHVPnIGr+SQ5TlbUTjyTwY02zYTIUcEMIWVRTMm8hc2mGZufZts8lbgLcc4f/deTAR0/hnVEqMwwruRf9Rv5B8nx8lZ0a2IrNNkg09hwPUPlET54NyKipF/Vwy4TaAx58/M0wB+I7DvwwA+3NS5FQof4sz5+/ebhy2TFsaSn/e37TInE4BagnZhe/ZnSNddGvlXdf6G9ski/7zfyZaP9uGsfcipmAHNd1xEBKa8nYuZDltvH1Lq6da0N45vf8cf+5l6Ra6DDhV2C9XscY9JnD8ltEvF6d5mcJkUzj9y7MgHn2q0j9xu3kTccyb7KpnYN5qM/BWKeQV3MRQXrvMvj/zNKz4szt84Ptk8FPmX0fpVaR8DEwxLpzrRioIJ36hn5B9y/si8kW/wSmrgCJ1/K0/4uiUOjN/dP+O/1skA7RMq75ANyF7aRzr/JJqOKPzW4e2f7XpILfLPgi5E/k7C19peQvsY20ZR6qlQzCvktI9/fz7JqRiputGnq/ZhFrSLo/aRA4f0NVVrwZTBRO8y4Wv6nWhTsQSBiPxLOf+A8yeR8A1x/qGErzvRKYv8ZwKRv6F9nIvwl3ewfxPXMqmGApDRPpXVPul9NW8jsk95TUAxER6K/M2AYuxx38TM/RnFSV4KxbyCuwC6i0LkLx7YKSc6Nbu6DudvnFqI6qk94Zv+bXkTvsXI34o0g2ofThRPnuOsCNpjoinBIKWeQHLvZZ9SnTPVw/m7P1co8reWZ3QOirI8Rf49Iqo8yJqE76IJQfsQ2fSekHpaton7YfeZ/39wKSTZX1MZX3X+irFBryCvmPDNHcNU15849Tr/OFzbp/aEr1H7iISvRfsUks+5swlFlF7aR0TMVBr5J049TzrnTteu1Jnfi9CcBp+TBwKcv1DUuG9e8pjYifyrJ3zZm/B1bZB/5bFB2sckfD1Szzzybwbq/BVjA/P89Yr8jZOS6hQ38pf8cjKxKe/b1aVbSd6qCd8hIn9z+klR5jnKeHrRvkTn716DSYxKPbqPypjuxJYjbrdypxtM+AYi/8m2X4ne9izmIgcct2BbYjcyOwAjWe2P9nETvq2ILFVRvwlf+ZbW8gzGUl7bBLSks2Js0GtNXuObjNOXDtCNTjPKIlWqGN16N+ZCEtF6C4j9A4GL6U5cKdFnHEMkkrBdJ+ruegafKApHlF3mLO+RbYvN+coriJq692awMLSP6/xlbiKU8PWt2JX0GYr8Be3jxP5Rxvmn31Paxx3kQjCDVanOv8+Er6S+5OS57DiZZG8A6vwVY4dQ5O9ORrIif0eOmEetovpkWv/FpUxCkX+vhO+iQOQrkUX+UucvOH/AoZoqSD0NlSXREZG/TC67mHHemoyzZC5J+Iaknn04f+Y8RxMzF3X+ztyE/hO+yb0r1fmLwcc6Ns4HTzjty3T+5np0GUeFYkj0nuFrc/7SWbk0TDeLWnPHNdNNI/8Az558Fn30iPwXtVvB/QZlOv+Jdh51u3b7pIUS7mBnAmQ5M9g3eMna+0BO0SSRv53v8N1DiX44/+TcZpZ18TfOB8f0OoiSCp0VnX8W+bfthK9EkPbhKjr/Ir2TU1SVTOwb6vwVY4NeUk/jjPPyDpy9chdonyxqFZF/x5Y55m3FZysKD9s6UzHyN7F/pvMXFMOkL/KXCd8Sp+JSMVUj/2kxcBLJiLsY+ceetyeJyUDkH1IqyRIbIZ2/dKhRBHT70PnHMRzO3yndMWDC1zh+95IyeW0lC/uHOn/F2CDn/AO0T/qwdQTnb5KOrvPPlCpW5B97KSX2RN7uZxfT3diSFYZQjPwl7ZOXnHDP2fI4Gwk3GpeJUllHyIWkfaSCpcD5x3lSOBT5+xK7gH82rDmnOVdohi+bN5h0EPNF/r4XC5NTkJRTLTr/WPTjnDf7v6SRv0IxHMzjF/K5kvbhlKYwHK+kfZKyyclnGfmbipYurMg/wP+7mO7EWFyF9kn/trwJ3+K6vjbHHPYq+53BzgyIhisHik4uieZT+7txOos4d7ohnb97LoNW5HfEIcpquizy90g9Q4XdfCWjDe3TbuUDWqLzF23SL6GEr/smY+owmXsUopFCk/GGhTp/xdig1wLuUudvHrxFWeTfFe3yY6Yczt8X+Yfq+ZSv+cp9Rf6URtmSSy9L+EY9In934lXuNMNVPa2lFzuxVTY6Zi7q/Ln49iQhKSZ3uzfyTwcR7wxfR+oZRSbyLw48oXkEccxZrsDXLpzwLVYETbYjXZA+2e6eNU/4NgN1/oqxQabzD3Dtuc4/T9oaByp1/vLh3u/QPr5IkjkfeHxRuA+VE77CNbTSSLY4wUoOPmnbKKzzB4pUjKzqGaJ9pHOf6cZJ5VARcbsL2fRK+LYiv5MP2W5z/n7ax5hMqRP3jb+hBeK7zNY1tYisNwxJuUmYCH/CyeFkfabX6F6q6vwViprQayFxk/uT9fl9nL/sR26f7vidPyAHnj44/ypST+EXonRxkHyCVTHyt2bqlnj/UI6DRDTu3k9Zr3+my5aCpSzhG5rhG4r8Q3SQ4fzLEr6yto+vpALgXy/ADFYy8nePl8l2iTzy99A+cT4Bz/09shm+yvkrFMMhc8A9Er4znTjjuHPOP49OyyL/kD/3KUFKdf6d/p1/i8hKLoZon0xd0kfkL/nnViDyl1LOaRP5p+cwORTZX5XIv+UxMqGsitvNm4fv9w1JPX0IRf6GomnJyN9pE8r5dOOiekm+TQAlkb/XyuGhzl8xNjDPZVDqKRK+ppKkeWBldCoduDXJq+t/+OW5Q0s6uhiI9onI4tInfbSPcDZlcOWXcrZpRuUUnL+H9hGR/3QnRPsYGaltg6RYCtvJ/i7P79ZWknAnefng2x7H+b0zu6VtZsWyUGDRiePCwvNx+pYWZbSPP4egkb9CMSR61vPPdP55iYYJH+0jI/+Ow/n3mD1sR+FhW2cqSj1lWBhVTPhmzqakW3fWrRX5B3T+LucvSxYwEs5/okVoR3ZuwpzLHeyCtI8zO7ntOn+P1FOWewZSyWrI+Xu8rVl/QNokdf7mmkKDf3Ltxcg/eZtIvhd0/iI/0QTU+SvGBtkkr+D+PPI3zn9Rq6j2kQ+49H/T3ThI5Zh2UlceHCjiJOHcT3kHIOX8rYSvf4ZviGbw2evaSiUJX1vtw5ac1HD+E62okJsw53IHu1DC150QlTn/VErKXOT83Ql+vlo62XkDnL9Z/tHY1HIGIFlUz8VMp8j590r4mvpISvsoFEMik3r2QfuYxTsMZdGKiouiG8x0ekf+VZZxNBF0tfIOjtpHRv5tE/nn7S1n04db8UX+xYSvzflLSWayNGTiALPchHO8O6+hJWSl1nZnhq9JbJvfLCns5tjv3P+I/BE+UK7zl1RUJGr7JCuWhQf0mTjOaLisz14J36yYnkb+CsVQyDn/EO2T/JWR/4TD+bei4upYBtNpRUv/uYu0D+BP+ubOv//IPxYTrXycv+VsPD4l5Gf8CV+/3YDk/JPvzMn+iVaU8uPFa1/sRP5RBG9U3IpQTvt4pJ457YPsOoK0j7OdKHljY7av3xv5h5y/h/Yxg0Uw4Wvetrw9Dg91/oqxATt/XRhnNN3JNemTjs7fRNc+hGrUADLh6zglT1/mLaOfSV7Gtl5VPcucDQBMeKJe2UeiFLK3GXQKnL9dU6eTcv4RFevdRFSs4imjbGmXS9mYRKp58/BN8nLfvGTdIReu2mciirJgwFL7RJT9Z2q37HvvYqbDhYSvyRHkb2Io7De2NgF1/oqxgY96kZC0TzbJK42+TVQbUVii6VbClPBN8pLnlDADTz/lHYCi2sfP+ctyAUWv4joo185Snb+Qcna6bBWPM1U988jfdpRmu0RLnEva5ZamMDRNmdrH2M+cX0fVyL/doqzvZEBKtsvj21EU1Pkb29x+szyCoJF8Nmt5B4UigGvu2YqbHnqqd0Mn6edClncwUayb8A2txwpUi/zdhV58A1F/kX/uGKKomtonU5d4fIpP4w7Yk8NCi7lYJbC7pryDkHqmkb+poy8d5WQrKji5SOj8pV2hyN/W+feifeDNJ/juQTsiMfjbtI95x2i3ytU+Jgfi2iQjfxdV1xoYFLqMo2Lk8aZLbgAAPPiRXyxt10/kbx52t7xDHPupGiA8Wcmck5kLC4b7HvBd+zsAgCWTvR9PK/I35R3SPifbxQjdTvgWEVpAxZQ+Jsuh27a7nP+yRW1rZu1MJ1X7pPSUvBUTbX/kn9E+hbVzyfleXtLZXcWtbJKXa8dEKxKRP7yRep7H8HbpjfxNVc9QDmZkZ/gS0elEdB0RrSeidUR0VrqdiOgTRLSRiG4lojObskEx2nhg2x7c+/iu0jZP7p6q3F/mEEJqnyzyz3X+bnmHLod53VCBMiDlvH2zPz3OYsOjOwAAzz5yebC/DMIxZBJKh/aRTrZXwjdI+3gi/yLt45Z3kIXdkv2TqZN3k6MTrSINY0X+Du1jRf6Z88+T6gW1j6PzLyvv4Kp9EtqnmPCWtX0yGwLe35S4ljC0T67zd/ZnM3yb8f5NRv5/BeBDzPwfRPSa9PtLAZwPYE367wUA/iH9q1BkeHL3FF72f36IJZMt3PHn5wXb3bZ5h/WdmYPSOF/kz8z4+x/eh6WTrby8QzfOqj0a52+omNDCHEC4Rk1ynuLyjoD/LWL9ph1YvriN4w9dmm0LLTnoFnaT5R189IyV8PVx/j0SvkRhtU+hqmeA809KKdt2JRJQuz+Z8G07CV9b/RNl5zTnKkb+uR3udbgocP5RhL1TM9k+OcmrYINvvcbUtiKnDzvh60b+DSd8m3T+DMCELgcCeDT9fAGAL3CSebmOiFYQ0ZHMvKVBW8YSzIw//Ndb8frnr8KLTjxkrs3pC9+4eTMAYO+0P5r+1FUbMdON8fEr7wWQcMZr//eV2LZ7Cq8742j87stOxDdv2Yw/fNUzs8FABv4798/go9+9GwceMIFP/GCj1XdS2M2O/G94MM8pSOVHqM6Pi3d95RY864hiJP+mS67HqUceiCMOXIyXPnMlzlh9EG59ZDued8wKyzm1iND1vLLYTjDn0mVdnMIkr4CzAcJLJ9rlHZJthfIOrs7fmuTFFucvcxNA8vv5E75Fu9zJXxMO53/Z+kdx8NJJq68Htu3BX3znTvzyc49Kr8M/exgovv1MtHLOX547ijyRf8cfGJgciEQh4euY80/XPpDZ2gSadP7vBvBdIvo/SOiln0+3Hw1gk2j3SLrNcv5EdBGAiwBg9erVDZq5cNGJGV+/+REcvWLxyDn/3VMJ733iyqXe/Zeu24SHntybfZ/uxtiWUkB3bNmJK+54Ap+66j689ZwTsXzxBIBc6REz43e/eDOu3bjN27dV3kGEd695zhH4zm2PZdHdRJrkm2xHiGMuTfhed/9TuO7+YlL69s07cfvmnQCSweOM1Qdh01N7sfbYgxxKB4BnbJFuQXLpUiljJXxZTipKcNrRyzMbQqtnDVLeoUWymmaeB/DRPu1WcTZvFOURtbTLXYXM5fwffmovPveTBwEAqw46AI88vQ8AcPE19+OklcvSY0qqehbUPjnnb5V3ICoc4+Z0rPsRET7yuufg+geewjdv2eyRevrtmZecPxFdSUS3e/5dAODtAP6AmY8B8AcAPtNP38x8MTOvZea1K1euHMbMsYX5DztdcZ3S+QSZvPPuFxTLyYcvyz4fsXyxlaCT6ho5xf+BbXuy7SuWTFgKj5mOoH1EFHjUgQekttkDw2QrwkQrKtTD8eHXzlxViEqz82aLnzMWTbTs4mUBD+CL/BMHnzu3bjDyT/6+7oxVeOvZJwAoUft4J3mFOf+YbW7e1PNPVD1F1VRI6mnuv6X2cSZ5mTkA0hxD0/3PX3w23vD8Vdn2q+5+AgAw2WpV1vm3o7zuv1XPP8rVPu4A5CLm5Pe48KzV+NgbnwcgV/uU5WCA5iZ5DRX5M/O5oX1E9AUA70q//iuAf0o/bwZwjGi6Kt2mqBnmFTT0H3I+Q64F64Mc0KQqZslky6rH7y4gAhSlnhOp8+7E3ewYl/YBgMUTLatPMwHMTFwqo30MJtv+ejWyX1MAzZZxBpw/7DZdzh18RvtIzj8u9sWi/1DCtyP4Z3dJRNf+zB5H6mnq2ySzpN2Er8f5R5QNsEWdv93ORV5B1KZ3fnj31vR8doL5n960Fp//6YP40b3bvDr/7FxEWW4i8kT+Zc+aOY7SRLihfUJ5FoN5Gfn3wKMAzkk/vxzAvennywG8KVX9vBDADuX7m8G0xwGOCozzLXuNNlgymU+GWrKohWkh1bTW3s3+Fvlm+YBLnb+kfUz5AVcGagaPMtrHIHFy/n0zaWG4TmwSo/m+ED9tz/BNa9Ck0WSQ9iH7WBbyz1bAEVnLOAYjf/t7MeEbY6JtEr7F38AdFIly5y/tkvMHAP+AZX53OVgBwL50gJ5oR1bkv2LJRPb/qKD2kclmcV8jygMJ01fZsybtkPcgn3Ed8vKjx/n/DoC/JaI2gP1I+XsA3wHwGgAbAewF8OYGbRhr+KLfUUFO2/Tp/Cfb2LFvxvvWkys+7O0TLbIW2pjpMmbiksi/Yxd9Sxw1V4r8TWEzH6ZlQblWVFDy+BCifexa+g7t41H75GUUyiN/i05ynP+0o3aSNrBI+Pp1/naEbj5nA2xk2yqv20dVTXXzipi+ezfpDMJE+TmLOn9pl6jqKdplE80CCV84duSyXBRyMC5GTu3DzNcCeL5nOwN4R1PnVeTIOP+S/5DzFb1oH+m8DxC0z9LJFmZEbZ4Zi/Pn7O+05fwjK8KfDkT+5rPL+ScUArC/ROop+whRODPdfLWryVZUWKIR8Ek+nWiS8yRiHvnnrZPI3+aYmSEif79tcuH3Kjp/09aifVLO36/ztyN/Y4+ZqOZSL3bkX3xbySN/v6qn+DvkfRY5f1tmmkX+TiTvuwcS1uAmZLk5jeQ/Tss7KPrGaEf+YdrHXRJwyYQd+Yc4fxaRv4xSJ1pRFsWbY8xxsrKmedgztU9k0z5lk7wMJlthiWGyfGSuJLIpHb9zdiN/w6Un9ESyvZDwdSJNRj7RKORo5ALuIZ2/O4nNSvjGiYIqr+fv0fkLb2T8raTW8n3+ev4+UOCaJlq2zr8s8nfzDb5lHN3Koj64M4IznX8P2qehwF/LOyxkTI9wwtc4fR/t4zqZAyzax+H8pfMXOnkr8m9HmOgI2qcTZ+UcFouBxTiLjPMXtE+Lh6d9rLIS7cibUJyICNPiGNmTWWvARP6ylr6BT+fPgnoI5R6z2aaUFzYrlHfoFCN/uZhLVtLZ5Cac2j7y9zE2Zglfx3FatE8oiQLj1Ivb3QSzajV4TAAAGKlJREFUpIeKOn9/vsGK/CslfOWbglwgxv49fNfQBDTyX8AwcsUQdTKfkc/WLJcUtiOyePmli9roCPpkxlp+Me9D+q1JD+c/3Y1BZC+6bT4aJ5dFpe0+Er7tMO0zLVRGCeefwzjcssg/SqmEbmxHqEWdvzk25eORO6bekX81qWdia05lMOfLOGa0jxX521SOW9OnoPOXtE9Z5E9+pZRLM8kqn8UZvrZdVuTvTPIqk1W7OY1M6pnp/APX0FDsr85/AWOUaR8Z3bv2y1mUCeWSPxwHGKlnx8P5B4r6+Dj/6U7CT8NxrtKeXOdPmGhRJZ1/r8jf2O2qX9xI2KCwgDsLnb9x0iydP4q0j9wWsK3fqp7GnlwWCru8Qw+dv/lsdP7yN44imx8vp338ZRwmHPrNivyd159Q5N9yBgXADjZcuG9ypg5T1k+FpH6dUOe/gDHKtI9dJMy2307WUvawtiLConaETsxZCWaf2seFHEDMg7ZnuoPJAPVS1Pkbzr/3fZ70FDDLr5OtgSWU8JVwI39ZLyZLzDq0T3ZNJipHvi3kR01VT0PlEPnr+bv2mP5m4hjduETn70gvi7SPO8NXRP6BuQnmGr0JX+cNTL4h9Ob80+uLCEZAXIXzt3IaIuEbKu8gbWsC6vwXMEY58pf8sUtbyeuZbNuRv3EWpiaQlTAOOv98ADELqOyZ6mJRuxUoI8DZceack+1qM3x76fzzOQQ2tRGO/KV9OZdu6fzZpn1cqSdz7mBCtI9bXtiUj5aY7sbWAjRy0ZVpIY+VuQmDSUd9k9E+7Qqcf8kkKQpc06TzBkbInXqp2icKlHfoU+dv1WHKxuJQ5K+0j6JPLITyDu5n97tL2Uw6zt9S+5TQPub5NhO5dk91sMiJ/CPXkTmRf2iRGIl2mc6/Izh/Z/3ekBLFGiAM7dNnwhfIB5Fg5C84fyDXqUvMdGJrHV5pw5RDZyWDVH6sq77JBruMEsmdPjmcf0g9lVxjWOpZ0PkHFFUTAZlpKxKTvKpw/s7/peS3Ktbzd4vrNRT4q/NfyMgi/wr68/kGGe33cv4mGUjIX9H3THcKbctoH/PwGnXPnqmE9nGllLJP46An2xSshlk8VxntExcoJYOQBt1tE4skomnadWkfTx+UDQiByN9x/kan7tov1VGRkHoaGay51+66CG0n8ncneZm5CHliOj9v2b0n+AcHtzKosRee9m0n35DP8A1Tgj64OQ3zW7mL67i/vdI+ir5hopBRpH3chUEkpq2EL1l8fUb7TKWRv2jLgdBcKj8s5+/y7umXbH1f4aDK5IYSvjIGBjNdzmcPu86/otonT/hK2idvE3tW8pL3JTS2yEVQjB3uf6uZLheksW7kX1bPX5470/mnA2wntpPY9qBcLvUMOU9b7ROO/N1JXjLha64gc/4lgVbxLc1ZXyHdvWjCXrtZ1T6KvpErXkbT+ZuHoSrtQ8glm1nkH/eO/CfblEXUZlLXrqkOFk1EBTUN4Kd93GgthMo6fyeazTXoPdQ+Wb0YUYDNpX0cZ1OJ8xc6/6SdfxlHd1Kc6S53/mTlJgzcyW9ujiOjsoxMVVx3eeRfbdEWEgXgfFU95TE+nX+1wm6S9kEh4WuuSSN/xdDoVSJhPmOmy1ialm3oSfuIh9BMvNpnOH9PYTcDSS14aR8RjRLlag2X9nHlpmVIVCb+fbKaaDHyN7aWR/6GS5cUiSv1dJ0NQ3L+/uuQM3yNPT6pp0X7EETkn/we2TKO7Or8nfyKI/XsxLG1rKP0z+Wcf3h/WO3jFHZr2ZG/+Zro/NmyoVznb9uc1WFy3mYm2+r8FUMiT/iOZuRvZu4WaB9H6pk5SkH7GGflq+1jYArC2bRP+uYw1U05/zwadHX+JkKblDb0gKsvt6/ZlnpKBMs7yDZRvsZwqKpnV9SSkZE/XAfknD/2cP6FhK8T+cso2chgswXcnci/oPN3In/zxhJFjvEYrLyDPIfpriV+awl3FTFf8n2gqp7sr+pZTPgq7aPoE6PO+Rvn7JZ4kA7dpn2KTtMq7+AEZebNQjqeRalUcbeT8G1HkXi1L0o9qzr/cs4/Fs7fbpNX3QxHhYaHN9GkrKhpYC0ekm5jcCHyd8/fcZx/FHkSvh12In+R8BVUWSuitLRBfuxE2z/JSw7mcu6CHfmX3HsR+bu33VL7oGyGb6C8A3k4/4q0TytK3hpiFnmMdN+itsP5a+Sv6BejrPaZ7sTZIi3um4scDCbbdi1+N2L1LeZisHRRKz0mf/ClVDGReiaf20KKWKjn344Kr+oh+BYtkba6+QSDcGE3O5pk9tXzz9tLmgHZ4CA5f2TXJJEtfC4ojwLtEztSz8ij80/vo0v7TLrlHcwg1M4jf0n7yLvQi/PPflvHqYYSvu6ksZDaxzfDtzNAeYfsxTVE+wR7HA7q/BcwjJOcicP/IecrZrqcRf5lk7xcvt11mr6qngZLF4nI31H7AMBkuwXz6LUF5eAqcvri/EsTvnmpadcBZMnIEg24LO/QklLPnlU9c2ohj/zt8xeknhEVEugu5y+remacfyrpjD1qH5cTT9rnTjVJ+Np2yrY+SKcuCwC6xxHyN4FSnX/k1/lXqurpDG5G7ePq/Iucv9I+ij4haZ+QzHG+ohOHaZ9pi/bJZ+cSFZ2jHDjcyD/j/AXlIKNDmfBtt6LCq715SPvh/CfbYZ0/kCeqQwnfAr8tvrqrQ2XLBRYKuzk0COdlGYyjcd+guoWEr6e8Q4dttY9P6pmWcfDV83cXO5H3wVyTb4JaaVVP5E59cdt/T01/oXvs0j4+nX/f9fwjWc/ffkNb5NqptI+iX5j/iMzhhdDnI0y9/jzydwu7hWf4Fjj/EspLcv7mOZZ1/cMJX5vzb/fB+fsif+ls9kwZ52+3MV97ST2N2sda/CW0gLtX7WPstM/vm+TlU/tMCqqsJXT5ks7KVrFy1T5WAjXfDhi1j+Tv87blVT3zfYvdyN/6HUhQa/Y9tovK2QlfcwV55B9+zuzrE+UdMsrIP8FPE76KvjFTUh9nPsPYmnP+JbV9BOUidf6+tm6kamiAScHnS4dQ4PwzOd+QOv8CrZAfuzedn1Dg/ANRaSHhy+7qUNSXzt9nkzlOHuMr75DV64/ye5nr/LuZ/V6df5sKCVFpR7ZGQS7sylAWGRPlEbXL+Vu0D4XvsVtOWiZ8Mxuc/xs+2G82yTWxSPj6lg5NjAtf3zBQ57+AIR3fKMk9jd1htU+gvAPZkbvbNqT2kZSOjM4WtfNJXu0oj9hDJZ2rwLeMozx2T8D5Z+UdClGh3cZw6TI6tev5F9UlidonfwtoRUUaKy/pnDt2X3mHZFZ0fi/NtWa1fQzt49P5O1LIpH0aFTsJX7cWfwgkbJfJ6EIf8toKtI89KMncgKFT+63q2Yoom4BYiPxnSeevK3ktYMiIeZTknq7zL5Z0Fk7DqatTTPiWcP6LhM5fRKsGttQzjzp9JZ2jqNqbVaJ2cbeJyH+qa2nJDapF/sjLOwT0+D51CQvex9TQcdU+RsUiHWSR9mHrWmTS2dL5RznlYSCltPb12pE/HNuB8sCYKM95LJ4oi/z9+n1js4EcgHz29pPwNW1z559G/iUDfJ1Q57+AUVYZcz5jOnP+vWmfdmTr/N0Hx17G0UbO+YsHOiAbbUV5YtnUCzLJ5YlWhKhiTsUf+efn2TPd8b5FmFIJbhRocf5OwtccZ9E+VsK3yPkzkho6E46Nbm2fiMgaTM152w4/7s7wNfc6SXbm/feifToxJwNfnJ8/O09paJwPUgcUnL9slb/1uf0VFo8vURzJWlIu3MHCtM3yIsHIvxn3r85/AcNy/iX/KecbTARkdPgF2kfkMibbeXkHoiJdYZV3cG7Bkkznb/PUBosmosxxSCmiqTuUOah2hCiuNrj6Er6Sqto73fUmj025Btcxya/GKcmJXG7CV6pLDBLO34T+aeTv2NDJOH9/5O+bmdyKirV9zL3ulfB1aZ9uzMlg7Iv8e3D+Mfsj/5DO380h2PX8/XMu+q7qaUX+ybZugPNvSu2jzn8BY9Q5/wMq1faxHVVR6imdvzPJS6h9fHzvZCvKnF5bTEKa7saW5G+yRehQ0WH74CvvYEX+aU0hF6a0gesH5PeWYx9g5J95G18VSTnDl5EMIoVJXjE7VTfJqhbqK0UdrOoZpbmJgtTTvl7AoX1E5O9ObguBEHb+xcJuxe2Jbf7IX67h64oBfHATxKZtVi02FPk3RPyo81/AkKWPR4n2yTj/iVBtH395B/Pd7stOdkoYGiDR+Sfb5AM62W5lD2Rb8PAzXbvIWCIV7f1mZVbncvXl0mGGIn+zcHrB0XkUJzPdWPDXOefNbKtLMqfCkv9PnOykM4h25PKPSKSYsRX52/JXY0+B9ilJ+LY8kb8ZzN11CKR15ZF/PviVJnxJlK4o0D7220xOqeVtqtT2CUf+6bHp/Vjkcv4NRf5DqX2I6A1EtIGIYiJa6+x7PxFtJKK7iejVYvt56baNRPS+Yc6vKMfIcv4dI/UM6PwLah/jzHpx/raDNnSLXGDFivzbOe3TjvK3gxlntmlVqadx6u6KVQXOv1182luRTaMYWJG/0Jr7Er7mWoqRv/3ZR/vErvMP0T6ysBvllEWe8CVvwtddxtGc3kT+nThO+sv4dnEPKkb+Rc6fvN/dnIxMsrsJX/N/yq375IN7/0zbPF/gj/ybwrBnuR3A6wBcIzcS0SkALgRwKoDzAPw9EbWIqAXgUwDOB3AKgF9P2yoawKg6f2ProokofUjCzn+yFeXFzqg4OcnS+cd2FJVLNSMv37uoHWXRmNT5z3Ria7Zp1cJumfN3nIm0ee9UgPOPEuWM6+gstQ9J+/L+TYRtnG2hvAMLqScbm4qcv5VfcFREvppEsrZPRvsIyWzXUW35Er55fSIzw9dcdz4QlgXGRPmgt8iJ/EM6f5djj5zB2qcMcx24D+7vbtq6iwTNVmG3oWgfZr4T8I68FwD4CjNPAXiAiDYCOCvdt5GZ70+P+0ra9o5h7Ahh+95pvOHTP22i65HApqf3gih5oH//X9ZnkfR8x74ZMyEomcD15esfxvc2PJ7tf3zn/nQxkZRKSZ/ppZPtwgO94dGdeOXHrs6Oa4uIS07SMlGhjPpkBLZkspX1vWuqg+WL25kyaLJNqLB2u9f5L1/chlnFKmbgqb3TOHTZosKxhm5wqQvJB0fCPuk8r7jjcbzyY1cLxY6xO+lLzlA2CXTX+e+b6RaKtt388NPZvZXVSDNFUJR/3rl/JstbmK6Nnd2Y7clhgi4yA+NkgRZKnG+H8yUrfYIruUiLG/m7cwVCUk83sWveBKKIMkdt7qW8pnZEmUNPjhV9UtJW9p/Z6TynTVVmaYrzPxrAdeL7I+k2ANjkbH+BrwMiugjARQCwevXqgYyIIsKaw5cNdOxCwJrDl+GlzzwMNz7wVDZ5aFTwwhMOwfOOWYHfe/kabHh0h7VvzeHL8PxjD8au/TM495TDsXzxBN573jNx3qlHgIjw7nPXYNNT+/CiEw/BD+563Drulaccjmvu2YZTj1qOF514CN569gl49pHLcfLhz8CuqQ7e/PPHI44Z92/bg7XHHoQVSybxuy89EW95yfE4aMkk3vzi4/D4zv143qoV+LnjDsZbzz4Bpx19IGa6jDeuXYVDli3CkQcuxhnHHIRdUzPYsn0/7t+2G885egWe3jsNAPjVM47GsYcswZ6pLl516uF4YOse/PpZq/HT+57ErqkZvPTkwzKb/+r1z8VxhyzFRIvw7COX41eedxQmWxFOOmwZrr5nK1516uFZ25c/6zD8bNN2dGPGa09PHrff+YUT8JP7tmVtTjnqQJz77KT/Xz9rNR7bsR/vfNlJWNSO8LZzTsTbzjkB39vwOE5YuRT/5QWrcc/ju3DfE7vx6I59OPWoA7N+fvOFx2LFkgnrdzlj9UF40QmH4gOveTbWPfQ0zl6zEssWtfHWs0/Apqf34llHLAcAvPKUI3D347sRM+M1px2JjU/sxiFLJ3HOM1finS87CccfuhRHrlgMAFixZDL7be/buicbwN70ouPwy887Cnc9tgsvPukQ/NkvnYL1m7bj3FMOx859Hdy/dTd2T3Ww5vBlOOmwZdixbwZveckJWLqojbOOPxgAcOwhS/BfX7Aa7YhwxPLFePGJh+LtLz0RZx13MP7o1c/EiSuXYfP2fTj1qOXZb7t4ooVXPPswbN87g0OWTuIvf+25+PxPHsT5px2Jt7xkO7ak9ykiwgkrl+LhJ/di30wXDz25F2ccc1B2r96w9hhMp5PiXnLSoQCAd5+7Bu0W4Y1rj8He6Q5WH7wUDz25p5CorgvUq+AXEV0J4AjPrg8w82Vpmx8C+ENmXpd+/ySA65j5i+n3zwD4j/S485j5t9PtvwngBcz8zjIb1q5dy+vWrat8UQqFQqEAiOgmZl7r29cz8mfmcwc452YAx4jvq9JtKNmuUCgUillCU2nlywFcSESLiOh4AGsA3ADgRgBriOh4IppEkhS+vCEbFAqFQhHAUJw/Ef0qgL8DsBLAt4loPTO/mpk3ENGlSBK5HQDvYOZuesw7AXwXQAvAJcy8YagrUCgUCkXf6Mn5zwco569QKBT9o4zz15LOCoVCMYZQ569QKBRjCHX+CoVCMYZQ569QKBRjiJFI+BLRVgAPDdHFoQC29Ww1+1C7+oPa1R/mq13A/LVtodl1LDOv9O0YCec/LIhoXSjjPZdQu/qD2tUf5qtdwPy1bZzsUtpHoVAoxhDq/BUKhWIMMS7O/+K5NiAAtas/qF39Yb7aBcxf28bGrrHg/BUKhUJhY1wif4VCoVAIqPNXKBSKMcSCdv7zabF4InqQiG4jovVEZBa9OZiIriCie9O/B/XqpyZbLiGiJ4jodrHNawsl+ER6D28lojNn2a4PEtHm9L6tJ6LXiH3vT+26m4he3aBdxxDRVUR0BxFtIKJ3pdvn9J6V2DWn94yIFhPRDUT0s9SuD6Xbjyei69PzfzUt64609PtX0+3XE9Fxs2zX54joAXG/Tk+3z9r//fR8LSK6hYj+Pf3e7P1i5gX5D0nJ6PsAnABgEsDPAJwyh/Y8COBQZ9tfAXhf+vl9AP5ylmw5G8CZAG7vZQuA1yBZhY0AvBDA9bNs1weRrBLntj0l/U0XATg+/a1bDdl1JIAz08/PAHBPev45vWclds3pPUuve1n6eQLA9el9uBTAhen2TwN4e/r5dwF8Ov18IYCvNnS/QnZ9DsDrPe1n7f9+er7/AeDLAP49/d7o/VrIkf9ZSBeLZ+ZpAGax+PmECwB8Pv38eQCvnY2TMvM1AJ6qaMsFAL7ACa4DsIKIjpxFu0K4AMBXmHmKmR8AsBHJb96EXVuY+eb08y4AdyJZk3pO71mJXSHMyj1Lr3t3+nUi/ccAXg7ga+l2936Z+/g1AK8gInsV9WbtCmHW/u8T0SoAvwjgn9LvhIbv10J2/kejuFh82YPRNBjA94joJkoWpweAw5l5S/r5MQCH+w+dFYRsmQ/38Z3pa/clghqbE7vSV+wzkESN8+aeOXYBc3zPUgpjPYAnAFyB5C1jOzN3POfO7Er37wBwyGzYxczmfn04vV9/Q0SLXLs8NteNjwN4L4A4/X4IGr5fC9n5zze8hJnPBHA+gHcQ0dlyJyfvcPNCdzufbAHwDwBOBHA6gC0APjpXhhDRMgBfB/BuZt4p983lPfPYNef3jJm7zHw6knW6zwLwrNm2wQfXLiI6DcD7kdj3cwAOBvDHs2kTEf0SgCeY+abZPO9Cdv5li8jPOph5c/r3CQDfRPJAPG5eI9O/T8yVfSW2zOl9ZObH0wc2BvCPyGmKWbWLiCaQONgvMfM30s1zfs98ds2Xe5bash3AVQBehIQ2MUvHynNndqX7DwTw5CzZdV5KnzEzTwH4LGb/fr0YwK8Q0YNI6OmXA/hbNHy/FrLznzeLxRPRUiJ6hvkM4FUAbk/t+a202W8BuGwu7EsRsuVyAG9KlQ8vBLBDUB2Nw+FYfxXJfTN2XZgqH44HsAbADQ3ZQAA+A+BOZv6Y2DWn9yxk11zfMyJaSUQr0s8HAHglknzEVQBenzZz75e5j68H8IP0TWo27LpLDOCEhFeX96vx35GZ38/Mq5j5OCR+6gfM/F/R9P2qM1s93/4hydbfg4Rv/MAc2nECEpXFzwBsMLYg4em+D+BeAFcCOHiW7PkXJHTADBIu8S0hW5AoHT6V3sPbAKydZbv+OT3vrel/+iNF+w+kdt0N4PwG7XoJEkrnVgDr03+vmet7VmLXnN4zAM8FcEt6/tsB/Jl4Dm5Akmj+VwCL0u2L0+8b0/0nzLJdP0jv1+0AvohcETRr//eFjS9FrvZp9H5peQeFQqEYQyxk2kehUCgUAajzVygUijGEOn+FQqEYQ6jzVygUijGEOn+FQqEYQ6jzVygUijGEOn+FQqEYQ/xf2g+2I8Iy68kAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "u2HaGRVEYGQS" + }, + "source": [ + "## Testing\n", + "The testing result will be the average reward of 5 testing" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 286 + }, + "id": "5yFuUKKRYH73", + "outputId": "3ad20d21-4c56-47b7-e617-d71e4211aed7" + }, + "source": [ + "fix(env, seed)\n", + "agent.network.eval() # set the network into evaluation mode\n", + "NUM_OF_TEST = 5 # Do not revise this !!!\n", + "test_total_reward = []\n", + "action_list = []\n", + "for i in range(NUM_OF_TEST):\n", + " actions = []\n", + " state = env.reset()\n", + "\n", + " img = plt.imshow(env.render(mode='rgb_array'))\n", + "\n", + " total_reward = 0\n", + "\n", + " done = False\n", + " while not done:\n", + " action, _ = agent.sample(state)\n", + " actions.append(action)\n", + " state, reward, done, _ = env.step(action)\n", + "\n", + " total_reward += reward\n", + "\n", + " img.set_data(env.render(mode='rgb_array'))\n", + " display.display(plt.gcf())\n", + " display.clear_output(wait=True)\n", + " \n", + " print(total_reward)\n", + " test_total_reward.append(total_reward)\n", + "\n", + " action_list.append(actions) # save the result of testing \n" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "-207.9114975585693\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAfbklEQVR4nO3deXRV9b338fc3AwmEMAdImBLaUGSQKUK01CKrXi1V4fqwFHutFvXSKq1Dr4reZ7WlvbUWa/Vely590Dq2FyuK1NV6L1pEqlUqoBEBUQaZwhAgTCFkOMn3+ePs2AiEjIeTnXxea+2VvX97ON9fzs4nO7/sc465OyIiEh4J8S5AREQaR8EtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIhE7PgNrOLzewTM9tkZnfF6nFERNobi8V93GaWCHwKXAjsBFYCV7n7+hZ/MBGRdiZWV9zjgU3uvsXdK4DngakxeiwRkXYlKUbH7QfsqLW8E5hQ18ZmppdvioicwN3tVO2xCu56mdksYFa8Hl9EJKxiFdyFwIBay/2Dts+5+3xgPuiKW0SkMWI1xr0SyDWzHDPrAMwAXonRY4mItCsxueJ294iZ/QBYAiQCT7r7ulg8lohIexOT2wEbXYSGSkRETlLXPyf1ykkRkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iETLM+c9LMtgJHgSog4u55ZtYD+AOQDWwFrnD3g80rU0REarTEFfcF7j7a3fOC5buApe6eCywNlkVEpIXEYqhkKvBMMP8MMC0GjyEi0m41N7gdeM3MVpvZrKCtj7vvDub3AH2a+RgiIlJLs8a4gYnuXmhmvYHXzWxD7ZXu7mbmp9oxCPpZp1onIiJ1M/dT5mrjD2Q2FygB/hWY5O67zSwTeNPdv1LPvi1ThIhIG+Ludqr2Jg+VmFmamaXXzAP/BKwFXgGuDTa7FvhjUx9DRERO1uQrbjMbDLwcLCYB/+3u95hZT+AFYCCwjejtgMX1HEtX3CIiJ6jrirvFhkqaQ8EtInKyFh8qERGR+FBwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGTqDW4ze9LMisxsba22Hmb2upltDL52D9rNzB4ys01mtsbMxsayeBGR9qghV9xPAxef0HYXsNTdc4GlwTLAN4HcYJoFPNoyZYqISI16g9vd/woUn9A8FXgmmH8GmFar/VmPWgF0M7PMlipWRESaPsbdx913B/N7gD7BfD9gR63tdgZtJzGzWWa2ysxWNbEGEZF2Kam5B3B3NzNvwn7zgfkATdlfRKS9auoV996aIZDga1HQXggMqLVd/6BNRERaSFOD+xXg2mD+WuCPtdqvCe4uyQcO1xpSERGRFmDupx+lMLMFwCSgF7AX+CmwGHgBGAhsA65w92IzM+BhonehlAIz3b3eMWwNlYiInMzd7VTt9Qb3maDgFhE5WV3BrVdOioiEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGTqDW4ze9LMisxsba22uWZWaGYFwTSl1rq7zWyTmX1iZhfFqnARkfaqIR8WfD5QAjzr7iOCtrlAibvff8K2w4AFwHggC/gLMMTdq+p5DH3mpIjICZr8mZPu/leguIGPMxV43t3L3f0zYBPREBcRkRbSnDHuH5jZmmAopXvQ1g/YUWubnUHbScxslpmtMrNVzahBRKTdaWpwPwp8CRgN7AZ+09gDuPt8d89z97wm1iAi0i41Kbjdfa+7V7l7NfA4/xgOKQQG1Nq0f9AmIiItpEnBbWaZtRb/Gai54+QVYIaZpZhZDpALvNe8EkVEpLak+jYwswXAJKCXme0EfgpMMrPRgANbge8BuPs6M3sBWA9EgNn13VEiIiKNU+/tgGekCN0OKCJykibfDigiIq2LgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZOoNbjMbYGbLzGy9ma0zs1uC9h5m9rqZbQy+dg/azcweMrNNZrbGzMbGuhMiIu1JQ664I8C/ufswIB+YbWbDgLuApe6eCywNlgG+SfTT3XOBWcCjLV61iEg7Vm9wu/tud38/mD8KfAz0A6YCzwSbPQNMC+anAs961Aqgm5lltnjlIiLtVKPGuM0sGxgD/B3o4+67g1V7gD7BfD9gR63ddgZtJx5rlpmtMrNVjaxZRKRda3Bwm1ln4CXgVnc/UnuduzvgjXlgd5/v7nnunteY/URE2rsGBbeZJRMN7d+7+6KgeW/NEEjwtShoLwQG1Nq9f9AmIiItoCF3lRjwW+Bjd3+g1qpXgGuD+WuBP9Zqvya4uyQfOFxrSEVERJrJoqMcp9nAbCLwFvARUB00/zvRce4XgIHANuAKdy8Ogv5h4GKgFJjp7qcdxzazRg2ziIi0B+5up2qvN7jPBAW3iMjJ6gpuvXJSRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIdOQDwseYGbLzGy9ma0zs1uC9rlmVmhmBcE0pdY+d5vZJjP7xMwuimUHRMLALDHeJUgb0pAPC84EMt39fTNLB1YD04ArgBJ3v/+E7YcBC4DxQBbwF2CIu1ed5jH0mZPSZmVnj6dv36GsWPEcoFNdGq6uz5xMasCOu4HdwfxRM/sY6HeaXaYCz7t7OfCZmW0iGuLvNrpqkZDLzh7PV8feRGJCEvu+tJnNm99B4S3N1agxbjPLBsYAfw+afmBma8zsSTPrHrT1A3bU2m0npw96EQB++cvvMW8ejBgBw4ZBVla8K2oOC0L7Rgb3mETPTrlk9h1GQsLpf+QmTZrE009/hSlTYPhwGDoUEjXKIieo94q7hpl1Bl4CbnX3I2b2KPAfRC8f/gP4DXBdI443C5jVuHKlLRs5cjCZmTB5cnR5925Yvz46/7//C5s2gTvs2QNVdQ68tQ79+o0kf8z1DO4xmdLK/Xy4/XlWv7+Q6urTF56RkcH48SUMHx5djkTgnXegshJ27oTFi6Pthw/D0aMx7oS0Wg0KbjNLJhrav3f3RQDuvrfW+seBPwWLhcCAWrv3D9q+wN3nA/OD/fW3o3zOglG9rKx/XHVfcEE0tKuqYMkSOH48Guy/+1386qxLSkpnvpQzkcz0UZRWHuCDbb9nxXtPc/z4oQYfo+Z7kJwMX/96dN4drr46Or92LXzySXT+2Wdh796TjyFtV0PuKjHgt8DH7v5ArfbMWpv9M7A2mH8FmGFmKWaWA+QC77VcydIeVVdHQzsSgdJSOHYsGt6tTUpKZ8af8x3GDv4OiQkd+HD7f7PivacoLT3Y7GPX/OKqqoKysuj34Nix6PdG2peGXHF/FfgO8JGZFQRt/w5cZWajiQ6VbAW+B+Du68zsBWA9EAFmn+6OEpHa3KMTRIcGCoIzbskS2LIluq64uHWGVYcOaZyT9y+MG3wNSQkpFGxfwLvvPdmk0K75PkQi8MYbUFEBhYXwyivR9SUlrfMXl5wZDbmr5G3gVLekvHqafe4B7mlGXdIOlZTAn/8cHf6oro6O4e7bF++qGq6qqoIjR/ZQVV3BgWOfsvmzv3LsWDHduw/g2LFiKiqONeg4BQXw+OOwbVv0+7B9e+v8RSXx0+B/TorE2vbtMHduvKtouqqqStatf5UkS6GkbB+HDu1i3Lgr6Nd3FLuL1rJy5YIGHeeBB2DVqhgXK6Gm4BZpQdXV1Wze9hZf/vL5DP/KFLK6jCU9JZPUpG7szFzD7t3r4l2itAEKbpEWYpZAbu75DB86hcz00fTsOIQES8LMyOkxiUMjtrF//xYqKzU4Lc2j4BZpAenpvcnOHk/e0O/Qv2v+54FdIzWpG5ndRjNw4Dg2b347jpVKW6DgFmmmzp0zOG/CDQzKOJe+nUeRYCf/WJkZfTuP4kuDz2P79tW66pZm0du6ijRRQkIiffoM5av5/8qoATPISh93ytCukZrUlaxuY8nOHn8Gq5S2SFfcIk2QkJDEsGEXMTz3EgZ1+yqdknvVu49ZAn06jyRnUD47drxPWZlesy5No+AWaaTOnXsxcsQljMz+P2Smj8Gs4X+4piZ1pV+PcQwcOI5PP30zdkVKm6ahEpFGysoayVmDvkVm+thGhTZEr7oz0s4ie8AEOnbsGqMKJewSEhJISqr7ulrBLdJIBw58xpGyXThNezljx6QeZPUcQ1bWyBauTMIuISGBQYMG8atf/YoRI0bUvd0ZrEmkTSgu3sGOPas5eHwL9X2C1KmYGRmdziJnUD6pqV1iUKGEUUJCArfeeisrV67k9ttvJ/E0b8Su4BZpJPcqNmx4nc1FyyiLNO1d/zol96Jfz7FkZQ1v4eokbJKSkpgyZQrLly/nF7/4BRkZGV94DcAp9zlDtYm0KYcP72bjZ8vo3W0og7pObMJYtzGg67mMHbGPo0f3sW/fphhVKq1ZXl4eDzzwAPn5+SQnJzd4P11xizTRtm0r2VX8PmWRw03aPykhhayuYxg8+NxGB7+E26BBg5g3bx6LFi3ia1/7WqNCG3TFLdJkFRWlbNn6LpndR5Pd7fwmhW+XlP4M6JPHZxkrKCraGIMqpbUwM/r06cMtt9zCzJkz6d27d71DInVRcIs0w7ZtK8nJib7UvWNy9/p3OEFSQip9u4wie9B49u//LAYVSmvQoUMHpkyZwuOPP06PHj3q/dDo+ujvM5FmqKwsY/Pmt9lTUtDoO0zcnUh1GZHq4yQlpTT56ktaLzPj4osvZvHixTz33HP06tWr2aENuuIWabYdOwrYlVNAn85n0ym5Z4P2qag6xsHjWyg8vJrthSv5+OPXqKqq/Hx9UlIKPXsOAozS0oMcPVoUo+olFsyMyZMnM2fOHM4991w6d+7cosevN7jNLBX4K5ASbP+iu/80+CDg54GewGrgO+5eYWYpwLPAOOAAcKW7b23RqkVakcrKUjZteZusHqPJ6X5BndtVexWllfspqdjDrkMfsH3XSrZseZdIpIJOnXqQltaTnJx8unQZSVqnCfTtORxwDhzZzFtvP055ud7bJAwGDhzIjBkz+NnPfkZqampMHqMhV9zlwGR3LzGzZOBtM/sf4EfAg+7+vJk9BlwPPBp8PejuXzazGcA84MqYVC/SSuza9RGFBz6gd9oI0jpkANGhkIqqo1R7NQeOf8Kxin3sKf6I4oPb2bp1JYcP7yI393y+lP01Oqf1Aowh/S/gcEfoUllGh8R0qr2S0spiEhP1x3Fr179/f2bOnMnVV19Nbm5uTIe+GvJhwQ6UBIvJweTAZODbQfszwFyiwT01mAd4EXjYzMyb8hIzkZAoLy9hy9Z36NdzDP0SJ3C0vJCjFbvZe3gtkepydu78kCNH9rBr1zrcqz7fr3v3AeRknE/3joMBp6r6CCUVxRQdK6ToyHoAiou3N/iDhuXMS01N5Y477uC6665j4MCBLTKGXZ8G/Ro3s0SiwyFfBh4BNgOH3D0SbLIT6BfM9wN2ALh7xMwOEx1O2V/X8TMzM4lEIuwL00d6S4t76623SEtLa9LLyFuDI0e2UlTyAaW+kwPFWyku3saOHe9TWVlGVVUE92o6dkz5wj4lJbvYe3wV+ysKAOfTt7Zy8NAhPvtsK9u2rQain2NZO+yldUhOTiY/P58bb7yRK6644rQvUW9pDQpuj541o82sG/AyMLS5D2xms4BZEB0TevbZZ/n2t7+t8G6HEhISuPzyy/nRj37E448/Hu9ymiU5OZWEhCQikQqqqirq3f7w4eMcO1ZFTk5fwKmoKMXdee655ygoGMzChQuprm7am1lJbJgZkyZN4sYbb+Siiy6iS5c4vN+MuzdqAn4C3EH0CjopaDsXWBLMLwHODeaTgu3sdMccN26cV1dX+5YtW/yGG25wokMxmtr41KFDB7/ssst8xYoVfvz4cZcvqqio8A0bNvgTTzzhU6dO9W7dusX9OWvvU8+ePX3x4sV+6NChmD//48aNc68rh+ta4f8I6gygWzDfEXgLuARYCMwI2h8DbgrmZwOPBfMzgBfqe4ygQHd3P3r0qM+ePduzsrLi/iRpis3UuXNnv+yyy/ydd97xysrKWJ//bUJ1dbUXFBT4iy++6BMnTvQJEyZ4UlJS3J/L9jJlZmb697//ff/oo4/O2HPe3OA+G/gAWAOsBX4StA8G3gM2EQ3xlKA9NVjeFKwfXN9j1A7uGu+9954PHDgw7k+Yppabunbt6lOnTvXly5d7dXV17M74dqCiosJfffVV//nPf+5nnXWWp6Wlxf35bYtTZmam33rrrb5mzZoz/hw3K7jPxHSq4K65wpg9e7auLEI+denSxadPn+5vvPGGV1VVteCpLdXV1R6JRPzPf/6z//rXv/b+/ft7enp63J/zsE89e/b0UaNG+fr16+N2zoYyuGtUVFT4HXfc4R07doz7k6mpcVNqaqrPmDHDly9f7uXl5S1wKsvpVFVVeUlJiS9fvtznzp3rM2fO9LS0NF34NHBKTk729PR0v+WWW3zjxo1eWloa178MQx3c7u5lZWX+/vvve25urptZ3J9gTaefEhMTffr06f7OO+/on45xVFZW5rt37/aHH37Yv/vd73rHjh3181PH+Tpp0iT/wx/+4Hv27Gk1/3cJfXDXKCws9F/+8pe6gmil04gRI3z69Om6S6QVikQi/umnn/qCBQt82rRpnpGREffzJd5Tzfj1hg0bzshdIo3VZoLbPTp0ct999/mQIUPi/sRrik4jRozwRx55xAsLC5tweko8rFu3zv/2t7/53Xff7eedd56npKTE/Tw6E1OHDh180qRJ/tprr/mGDRvi/TScVpsK7hrbt2/3CRMmxP1EaK+TmXlubq4/9thjXlRU1OjnT1qPSCTir7/+ut9///1+1llneZcuXeJ+frX0lJ2d7fPmzfMlS5Z4JBKJ97e8QU4X3ObuxFteXp6vWrWqUfu4Ozt37uSpp57i3nvvpaysLEbVSW2JiYkMHDiQ22+/nauvvpr09HS9j3Qb4e5UV1ezfPlytmzZwttvv81rr70GwIEDB6ioqP+VoK1Jeno6F1xwAZdeeimXXHIJffr0CdW5mpeXx6pVq05ZcGiDu0Z1dTVPPPEEt99+O0eP6m0vYyUxMZHs7Gx++MMfcv3115OWlhaqHwJpvMrKSioro+8RvnDhQrZv387SpUtZvXo1lZWVlJeXx7nCkyUkJDBgwACuu+46Jk6cyMSJE+nQoUO8y2qSNh3cAJFIhJ07dzJz5kzefPPNlitMMDNyc3O54YYbuO666+jRo4cCux07evQopaWlfPjhh7z88svs27ePxYsXAzVvhhWfPElKSmL69OmcffbZzJo1q02cp20+uGsUFRUxZ84cFi1axJEjR1qgsjMjPz+fzMzML7S9+eabHDx4ME4VRY0YMYKZM2dy1VVXnVSfCEBZWRnbtm0DoufskiVL2LNnD+++++4ZefwxY8YwZMgQ7rjjDkaOHBnaq+tTaTfBDdFxuoULF3LvvfdSUFDQIsdsaYMGDWLgwIEMHz6ca665hqFDh9K9+z8+aNbdWbduXZ1DP0899RQbNmw45bry8nJWrlzZrCufESNGcOONNzJt2jSysrKafBxpnw4ePMiGDRvYv38/v/nNb4hEIqxcubLFxsh79Ojx+c/OpZdeSp8+fVrkuK1NuwruGocOHeL6669n0aJFLXrcpsrJyWHQoEHcdtttDB06lCFDhsTkcY4fP86yZctO+VagZWVlzJs3j9LS0jr3v/nmm7n88svJyMiISX3S/lRVVbFs2TLKysrYvn07jzzyCFVVVWzcuLHBb1mbmJhIbm4u+fn53HTTTZxzzjkxrjr+2mVwAxw+fJjf/e53zJ07l/376/wch5jo2rUr3bp147bbbqNz585861vfonfv3phZ3Mbeam4lOp141idtX805ePz4cV566SUqKip45JFHKCoqIhKJUFT0xQ9F7tWrF1lZWdx5551Mnz6d5OTkM/IJM61Buw1uiJ4ob7zxBtdccw179uyJ2ZvSJycnk5KSwtVXX02/fv248MILOfvss0lNTVUQitTB3SkvL6e6upqDBw/yzDPPUFFRwfz587nyyiu56aabGDBgACkpKe3u56hdBzdET44DBw4wZ84cnnzyyRY7bmJiIp06dWLGjBnk5+dzySWX0L17d5KTk1vsMUTaG3enuLiYrl27kpTUfj8k+XTB3S6+K2ZGr169eOihh+jevTsLFixg165dTT7eN77xjc//gZeYmEhOTk67+fNNJNbMjJ49e8a7jFatXQR3jbS0NO6//34uv/xy7rzzTlasWEFVVf0fwjps2DCys7OZM2cOKSkpjBw5kk6dOp2BikVETtaugrvGeeedx9KlS/nxj3/Mgw8+SCQS+cL6Xr160bdvX2677TZ69+5NXl4effv2jVO1IiJf1C6DGyAlJYV77rmHsWPHct9997F371569erFzTffzKhRoxg7dqzusBCRVqne4DazVOCvQEqw/Yvu/lMzexr4OnA42PS77l5g0aT7L2AKUBq0vx+L4psrOTmZK6+8kmnTpkXfccusXf73WkTCpSFX3OXAZHcvMbNk4G0z+59g3R3u/uIJ238TyA2mCcCjwddWycxITU2NdxkiIg1W760QwVvDlgSLycF0unsIpwLPBvutALqZmd7oQkSkhTToHjYzSzSzAqAIeN3d/x6susfM1pjZg2aWErT1A3bU2n1n0CYiIi2gQcHt7lXuPhroD4w3sxHA3cBQ4BygBzCnMQ9sZrPMbJWZrdq3b18jyxYRab8a9aoRdz8ELAMudvfdwXBIOfAUMD7YrBAYUGu3/kHbicea7+557p6nNzQSEWm4eoPbzDLMrFsw3xG4ENhQM24d3EUyDVgb7PIKcI1F5QOH3X13TKoXEWmHGnJXSSbwjJklEg36F9z9T2b2hpllAAYUAN8Ptn+V6K2Am4jeDjiz5csWEWm/6g1ud18DjDlF++Q6tndgdvNLExGRU9E7I4mIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCxtw93jVgZkeBT+JdR4z0AvbHu4gYaKv9grbbN/UrXAa5e8apViSd6Urq8Im758W7iFgws1VtsW9ttV/QdvumfrUdGioREQkZBbeISMi0luCeH+8CYqit9q2t9gvabt/UrzaiVfxzUkREGq61XHGLiEgDxT24zexiM/vEzDaZ2V3xrqexzOxJMysys7W12nqY2etmtjH42j1oNzN7KOjrGjMbG7/KT8/MBpjZMjNbb2brzOyWoD3UfTOzVDN7z8w+DPr1s6A9x8z+HtT/BzPrELSnBMubgvXZ8ay/PmaWaGYfmNmfguW20q+tZvaRmRWY2aqgLdTnYnPENbjNLBF4BPgmMAy4ysyGxbOmJngauPiEtruApe6eCywNliHaz9xgmgU8eoZqbIoI8G/uPgzIB2YHz03Y+1YOTHb3UcBo4GIzywfmAQ+6+5eBg8D1wfbXAweD9geD7VqzW4CPay23lX4BXODuo2vd+hf2c7Hp3D1uE3AusKTW8t3A3fGsqYn9yAbW1lr+BMgM5jOJ3qcO8P+Aq061XWufgD8CF7alvgGdgPeBCURfwJEUtH9+XgJLgHOD+aRgO4t37XX0pz/RAJsM/AmwttCvoMatQK8T2trMudjYKd5DJf2AHbWWdwZtYdfH3XcH83uAPsF8KPsb/Bk9Bvg7baBvwXBCAVAEvA5sBg65eyTYpHbtn/crWH8Y6HlmK26w/wTuBKqD5Z60jX4BOPCama02s1lBW+jPxaZqLa+cbLPc3c0stLfumFln4CXgVnc/Ymafrwtr39y9ChhtZt2Al4GhcS6p2czsEqDI3Veb2aR41xMDE9290Mx6A6+b2YbaK8N6LjZVvK+4C4EBtZb7B21ht9fMMgGCr0VBe6j6a2bJREP79+6+KGhuE30DcPdDwDKiQwjdzKzmQqZ27Z/3K1jfFThwhkttiK8Cl5nZVuB5osMl/0X4+wWAuxcGX4uI/rIdTxs6Fxsr3sG9EsgN/vPdAZgBvBLnmlrCK8C1wfy1RMeHa9qvCf7rnQ8crvWnXqti0Uvr3wIfu/sDtVaFum9mlhFcaWNmHYmO239MNMCnB5ud2K+a/k4H3vBg4LQ1cfe73b2/u2cT/Tl6w93/hZD3C8DM0swsvWYe+CdgLSE/F5sl3oPswBTgU6LjjP833vU0of4FwG6gkuhY2vVExwqXAhuBvwA9gm2N6F00m4GPgLx413+afk0kOq64BigIpilh7xtwNvBB0K+1wE+C9sHAe8AmYCGQErSnBsubgvWD492HBvRxEvCnttKvoA8fBtO6mpwI+7nYnEmvnBQRCZl4D5WIiEgjKbhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCZn/D0FstkREZS2+AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Aex7mcKr0J01", + "outputId": "a36aaa35-ec20-4089-ddde-ab4742d3e90e" + }, + "source": [ + "print(np.mean(test_total_reward))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "-147.2620449863271\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "leyebGYRpqsF" + }, + "source": [ + "Action list" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "hGAH4YWDpp4u", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "24f547ee-6648-4d2a-f9c7-718b23e93251" + }, + "source": [ + "print(\"Action list looks like \", action_list)\n", + "print(\"Action list's shape looks like \", np.shape(action_list))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Action list looks like [[2, 2, 2, 2, 2, 2, 1, 2, 1, 2, 1, 2, 2, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 1, 2, 2, 2, 2, 2, 0, 3, 2, 3, 2, 3, 3, 2, 2, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 2, 3, 3, 2, 3, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 3, 2, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 3, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 2, 1, 2, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 1, 2, 2, 2, 1, 1, 2, 2, 1, 1, 1, 2, 2, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 2, 2, 3, 2, 3, 3, 2, 3, 2, 3, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3], [2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 1, 1, 2, 2, 1, 2, 1, 2, 1, 2, 2, 2, 1, 2, 1, 1, 2, 1, 2, 2, 2, 2, 2, 2, 1, 1, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 3, 3, 2, 3, 3, 3, 3, 3, 2, 3, 2, 3, 3, 3, 3, 2, 2, 3, 3, 3, 3, 2, 2, 2, 3, 3, 3, 2, 2, 3, 3, 3, 3, 2, 2, 3, 3, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1, 2, 1, 2, 2, 1, 2, 1, 2, 2, 2, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 1, 2, 1, 2, 1, 1, 0, 2, 1, 1, 2, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 2, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 3, 3, 3, 3, 3, 2, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 2, 3, 3, 2, 2, 2, 3, 3, 2, 3, 0, 2, 3, 2, 0, 2, 3, 3, 2, 3, 2, 3, 2, 3, 2, 2, 3, 2, 3, 2, 2, 3, 3, 2, 3, 2, 2, 3, 2, 2, 2, 2, 3, 3, 2, 2, 2, 2, 3, 2, 2, 0, 1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 2, 1, 2, 2, 1, 1, 1, 1, 2, 2, 1, 1, 1, 2, 1, 2, 1, 2, 2, 1, 1, 2, 2, 1, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 1, 2, 2, 2, 0, 2, 3, 2, 3, 3, 3, 2, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 2, 2, 3, 2, 3, 3, 3, 2, 3, 2, 3, 2, 2, 3, 2, 3, 2, 3, 2, 3, 3, 2, 2, 3, 2, 3, 3, 3, 2, 3, 3, 2, 2, 3, 2, 3, 2, 2, 3, 3, 2, 2, 2, 2, 2, 2, 0, 2, 1, 2, 2, 0, 1, 2, 2, 0, 2, 1, 1, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 2, 2, 1, 2, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 2, 2, 2, 1, 2, 0, 2, 0, 3, 2, 3, 2, 0, 2, 0, 2, 3, 3, 3, 2, 2, 3, 2, 3, 2, 3, 2, 3, 3, 3, 2, 3, 3, 3, 3, 2, 2, 3, 2, 2, 2, 3, 2, 2, 3, 2, 3, 2, 2, 3, 2, 2, 3, 2, 0, 2, 3, 2, 2, 2, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 2, 2, 2, 2, 2, 2, 0, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 2, 1, 2, 2, 1, 2, 2, 1, 2, 1, 1, 2, 2, 0, 2, 1, 2, 1, 1, 1, 2, 1, 2, 2, 1, 2, 1, 2, 2, 2, 2, 1, 2, 2, 1, 2, 2, 1, 2, 1, 2, 2, 2, 1, 1, 2, 3, 3, 2, 3, 2, 3, 2, 3, 2, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 3, 2, 3, 3, 2, 2, 2, 2, 3, 2, 3, 2, 3, 3, 2, 3, 2, 2, 3, 2, 3, 2, 2, 2, 3, 2, 0, 2, 1, 2, 1, 2, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 1, 1, 2, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 2, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2, 1, 2, 2, 2, 1, 1, 2, 2, 2, 1, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2, 1, 1, 2, 2, 1, 2, 2, 2, 0, 2, 2, 1, 2, 2, 2, 3, 0, 3, 2, 3, 3, 2, 3, 3, 2, 3, 3, 2, 2, 3, 3, 3, 3, 2, 3, 3, 3, 2, 2, 3, 2, 2, 3, 3, 3, 2, 2, 2, 3, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 1, 2, 2, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 2, 2, 1, 2, 1, 1, 2, 2, 1, 2, 1, 1, 2, 2, 2, 1, 2, 1], [0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 2, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 1, 1, 2, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 2, 2, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 2, 3, 2, 3, 2, 3, 3, 2, 2, 3, 3, 2, 3, 2, 3, 3, 2, 3, 2, 3, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 2, 3, 2, 3, 2, 2, 2, 2, 2, 2, 0, 2, 1, 2, 2, 0, 1, 2, 1, 2, 2, 1, 1, 2, 0, 2, 1, 2, 2, 2, 2, 2, 1, 1, 2, 1, 0, 2, 2, 2, 2, 1, 2, 2, 2, 1, 1, 2, 2, 2, 1, 2, 1, 2, 2, 2, 2, 1, 2, 1, 2, 1, 1, 2, 2, 1, 2, 2, 2, 3, 2, 3, 2, 2, 3, 2, 3, 2, 3, 3, 2, 3, 3, 3, 2, 3, 3, 3, 3, 2, 3, 3, 2, 2, 2, 2, 3, 2, 3, 3, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 0, 0, 2, 2, 1, 1, 2, 2, 2, 1, 1, 2, 2, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 0, 1, 2, 1, 1, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2, 1, 1, 2, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 2, 3, 3, 3, 2, 3, 3, 3, 2, 3, 3, 2, 2, 2, 3, 2, 3, 2, 2, 3, 3, 2, 3, 3, 3, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 2, 2, 1, 2, 1, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1], [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 3, 3, 2, 3, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 2, 1, 1, 1, 2, 1, 2, 1, 2, 2, 1, 2, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 2, 1, 2, 2, 1, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 1, 2, 2, 2, 3, 2, 2, 0, 2, 2, 2, 3, 2, 3, 2, 2, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 2, 3, 3, 3, 3, 2, 3, 3, 2, 3, 3, 3, 2, 2, 3, 2, 3, 3, 2, 3, 2, 3, 2, 2, 2, 3, 2, 3, 2, 3, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 2, 1, 1, 1, 2, 2, 1, 2, 1, 1, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]]\n", + "Action list's shape looks like (5,)\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n", + " return array(a, dtype, copy=False, order=order)\n" + ], + "name": "stderr" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fNkmwucrHMen" + }, + "source": [ + "Analysis of actions taken by agent" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "WHdAItjj1nxw", + "outputId": "20aea8b1-e011-4397-b775-7b4c4b593871" + }, + "source": [ + "distribution = {}\n", + "for actions in action_list:\n", + " for action in actions:\n", + " if action not in distribution.keys():\n", + " distribution[action] = 1\n", + " else:\n", + " distribution[action] += 1\n", + "print(distribution)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "{2: 1144, 1: 516, 0: 30, 3: 501}\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ricE0schY75M" + }, + "source": [ + "Saving the result of Model Testing\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "GZsMkGmIY42b", + "outputId": "c47a3123-eb1b-4dc1-f1b2-7a09a82cd8a6" + }, + "source": [ + "PATH = \"Action_List.npy\" # Can be modified into the name or path you want\n", + "np.save(PATH ,np.array(action_list)) " + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:2: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n", + " \n" + ], + "name": "stderr" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "asK7WfbkaLjt" + }, + "source": [ + "### This is the file you need to submit !!!\n", + "Download the testing result to your device\n", + "\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 34 + }, + "id": "c-CqyhHzaWAL", + "outputId": "38653c82-673e-4f90-8746-3a0424fe3aca" + }, + "source": [ + "from google.colab import files\n", + "files.download(PATH)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "\n", + " async function download(id, filename, size) {\n", + " if (!google.colab.kernel.accessAllowed) {\n", + " return;\n", + " }\n", + " const div = document.createElement('div');\n", + " const label = document.createElement('label');\n", + " label.textContent = `Downloading \"${filename}\": `;\n", + " div.appendChild(label);\n", + " const progress = document.createElement('progress');\n", + " progress.max = size;\n", + " div.appendChild(progress);\n", + " document.body.appendChild(div);\n", + "\n", + " const buffers = [];\n", + " let downloaded = 0;\n", + "\n", + " const channel = await google.colab.kernel.comms.open(id);\n", + " // Send a message to notify the kernel that we're ready.\n", + " channel.send({})\n", + "\n", + " for await (const message of channel.messages) {\n", + " // Send a message to notify the kernel that we're ready.\n", + " channel.send({})\n", + " if (message.buffers) {\n", + " for (const buffer of message.buffers) {\n", + " buffers.push(buffer);\n", + " downloaded += buffer.byteLength;\n", + " progress.value = downloaded;\n", + " }\n", + " }\n", + " }\n", + " const blob = new Blob(buffers, {type: 'application/binary'});\n", + " const a = document.createElement('a');\n", + " a.href = window.URL.createObjectURL(blob);\n", + " a.download = filename;\n", + " div.appendChild(a);\n", + " a.click();\n", + " div.remove();\n", + " }\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "download(\"download_899cef0d-01bc-40fd-a501-1573f2382641\", \"Action_List.npy\", 4689)" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "seT4NUmWmAZ1" + }, + "source": [ + "# Server \n", + "The code below simulate the environment on the judge server. Can be used for testing." + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "U69c-YTxaw6b", + "outputId": "65dd4cf4-e667-469d-a7dc-34f6a053e1d0" + }, + "source": [ + "action_list = np.load(PATH,allow_pickle=True) # The action list you upload\n", + "seed = 543 # Do not revise this\n", + "fix(env, seed)\n", + "\n", + "agent.network.eval() # set network to evaluation mode\n", + "\n", + "test_total_reward = []\n", + "if len(action_list) != 5:\n", + " print(\"Wrong format of file !!!\")\n", + " exit(0)\n", + "for actions in action_list:\n", + " state = env.reset()\n", + " img = plt.imshow(env.render(mode='rgb_array'))\n", + "\n", + " total_reward = 0\n", + "\n", + " done = False\n", + "\n", + " for action in actions:\n", + " \n", + " state, reward, done, _ = env.step(action)\n", + " total_reward += reward\n", + " if done:\n", + " break\n", + "\n", + " print(f\"Your reward is : %.2f\"%total_reward)\n", + " test_total_reward.append(total_reward)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/torch/__init__.py:422: UserWarning: torch.set_deterministic is deprecated and will be removed in a future release. Please use torch.use_deterministic_algorithms instead\n", + " \"torch.set_deterministic is deprecated and will be removed in a future \"\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Your reward is : -29.53\n", + "Your reward is : -36.44\n", + "Your reward is : -194.16\n", + "Your reward is : -268.27\n", + "Your reward is : -207.91\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAc+klEQVR4nO3de3RU9b338feXJCRcwv1iCAmgDQcRFSFC9NAWWbVaq8JxsRR7qBbx0Cqtlx4ROc9qS/sstVqrz3HJ0getFts+qHgry9qDFpDWWpWLEQGxIuUWgSg3wZDLZL7PH7NDRyD3mUx25vNaa6/Z+7f3zP7+MptPNr/Zk23ujoiIhEenVBcgIiLNo+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQSVpwm9nFZvaBmW0xszuStR8RkXRjybiO28wygL8DFwK7gNXA1e6+KeE7ExFJM8k64x4HbHH3re5eDTwFTE7SvkRE0kpmkl43H9gZt7wLGF/fxmamr29KQnXt2pvMzByqKj+jqrqCnJzudO8ygKyMbq163dpoFZ9Xfcrnn+8jKyuHrl17E4nUUFGxH/dogqoXiXF3O1l7soK7UWY2C5iVqv1Lx9Wjxylc8JVbGN7vYjaVv8DyFfczfvy3KR46k9zsvFa99oGj/2D1R4+xZs3TnF8yk7OHTONo5ABvblzI+vVLE9QDkYYla6ikDCiIWx4ctB3j7gvdvdjdi5NUg6Sp006bQH7PsThRKioOEI3WJnwf1dUV7Cx7h4OV2+nXdThDBo8jN3dAwvcjcjLJCu7VQJGZDTOzzsA0QKcjknS9euVTOKiYXjlD2H34Xbb+429UVx9Nyr527lxH2f61RKKVDOo5hmHDSpKyH5HjJSW43T0CfB9YBrwPPOPuG5OxL5E6mZmd+dJpX2Fwr3Oprv2cj/etY9eu0qTtr7q6gq3/+Bt7Dr9Lny6nMiT/XHr0OCVp+xOpk7Qxbnd/GXg5Wa8vcrz8/LMYMexiemTns2H3s6x751lqa2uA2Oc7tV5DJFrZqn1EveYLyzt2rOXjYaWckjuaQb3GMGRIMe+991Kr9iHSmJR9OCmSSBkZWZx22gTyuo9m/9GtbC97i0OHdgdrnX37trGl+zKMk35IfxwDTn6hkxPl00+3HVuORKr4aOvr5PU+m/we4xiSP45t297m8OHy1nZJpF4KbukQCgrOYVDv0WR2yubjg++wffvaL6zftOkVPvrorwnZV1XVkS8s79xZyu5T3+WU7mfTs9tgCgvHsHHjMuoLf5HWUnBLh1BdXcHH+96lIrKPHR+v5tChj7+wPhqNcPTooaTsOxKpYuOmP9I1py+fHf44ONNXaEvyJOUr780uQl/AkQTIzu5Obm5/Dh7cTSTSurHslujZcxAVFQepqalo831Lx1TfF3AU3CIi7VR9wa0/6yoiEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIh06q/x21m24DDQC0QcfdiM+sDPA0MBbYBV7r7gdaVKSIidRJxxn2Bu4929+Jg+Q5gubsXAcuDZRERSZBkDJVMBhYF84uAKUnYh4hI2mptcDvwipmtNbNZQdtAd6+7S+seYGAr9yEiInFae8/JCe5eZmYDgFfNbHP8Snf3+u5uEwT9rJOtExGR+iXs1mVmNh84AvwHMNHdd5tZHvCau/9LI8/VrctERI6T8FuXmVk3M8utmwe+DmwAlgLXBptdC/y+pfsQEZETtfiM28xOBV4IFjOB/+fud5pZX+AZoBDYTuxywP2NvJbOuEVEjqO7vIuIhIzu8i4i0kEouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyjQa3mT1uZuVmtiGurY+ZvWpmHwaPvYN2M7MHzWyLma03szHJLF5EJB015Yz718DFx7XdASx39yJgebAM8A2gKJhmAQ8npkwREanTaHC7+5+B/cc1TwYWBfOLgClx7U96zJtALzPLS1SxIiLS8jHuge6+O5jfAwwM5vOBnXHb7QraTmBms8xsjZmtaWENIiJpKbO1L+DubmbeguctBBYCtOT5IiLpqqVn3HvrhkCCx/KgvQwoiNtucNAmIiIJ0tLgXgpcG8xfC/w+rv2a4OqSEuBQ3JCKiIgkgLk3PEphZouBiUA/YC/wE+BF4BmgENgOXOnu+83MgIeIXYVSAcxw90bHsDVUIiJyIne3k7U3GtxtQcEtInKi+oJb35wUEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyDQa3Gb2uJmVm9mGuLb5ZlZmZqXBdEncunlmtsXMPjCzi5JVuIhIumrKzYK/AhwBnnT3UUHbfOCIu9933LYjgcXAOGAQ8CdguLvXNrIP3XNSROQ4Lb7npLv/GdjfxP1MBp5y9yp3/wewhViIi4hIgrRmjPv7ZrY+GErpHbTlAzvjttkVtJ3AzGaZ2RozW9OKGkRE0k5Lg/th4DRgNLAb+GVzX8DdF7p7sbsXt7AGEZG01KLgdve97l7r7lHgUf45HFIGFMRtOjhoExGRBGlRcJtZXtzivwF1V5wsBaaZWbaZDQOKgLdbV6KIiMTLbGwDM1sMTAT6mdku4CfARDMbDTiwDfgugLtvNLNngE1ABJjd2BUlIiLSPI1eDtgmRehyQBGRE7T4ckAREWlfFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIdNocJtZgZmtNLNNZrbRzG4O2vuY2atm9mHw2DtoNzN70My2mNl6MxuT7E6IiKSTppxxR4D/dPeRQAkw28xGAncAy929CFgeLAN8g9jd3YuAWcDDCa9aRCSNNRrc7r7b3dcF84eB94F8YDKwKNhsETAlmJ8MPOkxbwK9zCwv4ZWLiKSpZo1xm9lQ4BzgLWCgu+8OVu0BBgbz+cDOuKftCtqOf61ZZrbGzNY0s2YRkbTW5OA2s+7Ac8At7v5Z/Dp3d8Cbs2N3X+juxe5e3JzniYikuyYFt5llEQvt37n780Hz3rohkOCxPGgvAwrinj44aBMRkQRoylUlBvwKeN/d749btRS4Npi/Fvh9XPs1wdUlJcChuCEVERFpJYuNcjSwgdkE4C/Ae0A0aP4vYuPczwCFwHbgSnffHwT9Q8DFQAUww90bHMc2s2YNs4iIpAN3t5O1NxrcbUHBLSJyovqCW9+cFBEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMg05WbBBWa20sw2mdlGM7s5aJ9vZmVmVhpMl8Q9Z56ZbTGzD8zsomR2QEQk3TTlZsF5QJ67rzOzXGAtMAW4Ejji7vcdt/1IYDEwDhgE/AkY7u61DexD95wUETlOi+856e673X1dMH8YeB/Ib+Apk4Gn3L3K3f8BbCEW4iIikgDNGuM2s6HAOcBbQdP3zWy9mT1uZr2DtnxgZ9zTdtFw0IsAcNdd3+Wee2DUKBg5EgYNSnVFbW/ixIn8+tf/wiWXwBlnwIgRkJGR6qqkvcls6oZm1h14DrjF3T8zs4eB/w148PhL4LpmvN4sYFbzypWO7MwzTyUvDyZNii3v3g2bNsXm/+d/YMsWcIc9e6C23oG3cOvfvz/jxh3hjDNiy5EIvPEG1NTArl3w4oux9kOH4PDh1NUpqdWk4DazLGKh/Tt3fx7A3ffGrX8UeClYLAMK4p4+OGj7AndfCCwMnq8xbjnGglG9QYP+edZ9wQWx0K6thWXL4OjRWLD/9repqzOZ6n4GWVnw1a/G5t1h+vTY/IYN8MEHsfknn4S9e098Dem4mnJViQG/At539/vj2vPiNvs3YEMwvxSYZmbZZjYMKALeTlzJko6i0VhoRyJQUQGffx4L73RS94urthYqK2M/g88/j/1sJL005Yz7X4FvA++ZWWnQ9l/A1WY2mthQyTbguwDuvtHMngE2ARFgdkNXlIjEc49NEBsaKA2OuGXLYOvW2Lr9+zt+WNX9HCIRWLECqquhrAyWLo2tP3Ik/X5xyT81Gtzu/jpwsktSXm7gOXcCd7aiLklDR47AH/4QG/6IRmNjuJ98kuqq2l5pKTz6KGzfHvs57NjR8X9RSfM0+cNJkWTbsQPmz091Fal3//2wZk2qq5D2TF95FxEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkdB23iEg706lTJzp1qv+8WmfcIiLtRKdOnRgyZAg///nPGTVqVP3btWFNIiJSj06dOnHLLbewevVqbrvtNjIa+EPsGioREUmhzMxMvv71rzNv3jzGjh1Lly5dGn9OG9QlIiInUVxczP33309JSQlZWVlNfp6CW0SkjQ0ZMoQbb7yRq6++moKCgsafcBwFt4hIGzAzBg4cyM0338yMGTMYMGAAZie9iXujFNwiIknWuXNnLrnkEh599FH69OnT4KV+TaHgFhFJEjPjoosu4qabbuLLX/4y3bt3T8jrKrhFRBLMzJg0aRJz587lvPPOS1hg12k0uM0sB/gzkB1s/6y7/yS4EfBTQF9gLfBtd682s2zgSWAssA+4yt23JbRqEZF2qrCwkGnTpvHTn/6UnJycpOyjKWfcVcAkdz9iZlnA62b2R+CHwAPu/pSZPQLMBB4OHg+4+5fMbBpwD3BVUqoXEWknBg8ezIwZM5g+fTpFRUUt/uCxKZpys2AHjgSLWcHkwCTgW0H7ImA+seCeHMwDPAs8ZGYWvI6ISIeSk5PDnDlzuO666ygsLGz1B49N0aQxbjPLIDYc8iVgAfARcNDdI8Emu4D8YD4f2Ang7hEzO0RsOOXT+l4/Ly+PSCTCJ+l4S2855i9/+QvdunUjnX/Hb9u2jUOHDpGRkUFtbW2qy5EGZGVlUVJSwg033MCVV17Z4FfUE61Jwe3utcBoM+sFvACMaO2OzWwWMAtiY0JPPvkk3/rWtxTeaahTp05cccUV/PCHP+TRRx9NdTntwm9+8xtKS0tZsmQJkUgkrX+ZtTdmxsSJE7nhhhu46KKL6NGjR9sX4e7NmoAfA3OInUFnBm3nAcuC+WXAecF8ZrCdNfSaY8eO9Wg06lu3bvXrr7/eiQ3FaOrgU+fOnf3yyy/3N998048ePeryRdXV1b5582Z/7LHHfPLkyd6rV6+Uv2fpPvXt29dffPFFP3jwYNLf/7Fjx7rXl8P1rfB/BnV/oFcw3wX4C3ApsASYFrQ/AtwYzM8GHgnmpwHPNLaPoEB3dz98+LDPnj3bBw0alPI3SVNypu7du/vll1/ub7zxhtfU1CT7+O8QotGol5aW+rPPPusTJkzw8ePHe2ZmZsrfy3SZ8vLy/Hvf+56/9957bfaetza4zwLeAdYDG4AfB+2nAm8DW4iFeHbQnhMsbwnWn9rYPuKDu87bb7/thYWFKX/DNCVu6tmzp0+ePNlXrVrl0Wg0eUd8GqiurvaXX37Zf/azn/npp5/u3bp1S/n72xGnvLw8v+WWW3z9+vVt/h63KrjbYjpZcNedYcyePVtnFiGfevTo4VOnTvUVK1Z4bW1tAg9tiUajHolE/A9/+IP/4he/8MGDB3tubm7K3/OwT3379vWzzz7bN23alLJjNpTBXae6utrnzJnjXbp0Sfmbqal5U05Ojk+bNs1XrVrlVVVVCTiUpSG1tbV+5MgRX7Vqlc+fP99nzJjh3bp104lPE6esrCzPzc31m2++2T/88EOvqKhI6f8MQx3c7u6VlZW+bt06LyoqcjNL+RusqeEpIyPDp06d6m+88YY+dEyhyspK3717tz/00EP+ne98x7t06aJ/P/UcrxMnTvSnn37a9+zZ024+dwl9cNcpKyvzu+66S2cQ7XQaNWqUT506VVeJtEORSMT//ve/++LFi33KlCnev3//lB8vqZ7qxq83b97cJleJNFeHCW732NDJvffe68OHD0/5G68pNo0aNcoXLFjgZWVlLTg8JRU2btzof/3rX33evHl+/vnne3Z2dsqPo7aYOnfu7BMnTvRXXnnFN2/enOq3oUEdKrjr7Nixw8ePH5/yAyFdJzPzoqIif+SRR7y8vLzZ75+0H5FIxF999VW/7777/PTTT/cePXqk/PhK9DR06FC/5557fNmyZR6JRFL9I2+ShoLb3J1UKy4u9jVr1jTrOe7Orl27eOKJJ7j77ruprKxMUnUSLyMjg8LCQm677TamT59Obm5uUv+YjrQddycajbJq1Sq2bt3K66+/ziuvvALAvn37qK6uTnGFzZObm8sFF1zAZZddxqWXXsrAgQNDdawWFxezZs2akxYc2uCuE41Geeyxx7jttts4fPhwgiuTOhkZGQwdOpQf/OAHzJw5k27duoXqH4E0X01NDTU1NQAsWbKEHTt2sHz5ctauXUtNTQ1VVVUprvBEnTp1oqCggOuuu44JEyYwYcIEOnfunOqyWqRDBzdAJBJh165dzJgxg9deey1xhQlmRlFREddffz3XXXcdffr0UWCnscOHD1NRUcG7777LCy+8wCeffMKLL74IxE6iUpUnmZmZTJ06lbPOOotZs2Z1iOO0wwd3nfLycubOncvzzz/PZ599loDK2kZJSQl5eXlfaHvttdc4cOBAiiqKGTVqFDNmzODqq68+oT4RgMrKSrZv3w7Ejtlly5axZ88e/va3v7XJ/s855xyGDx/OnDlzOPPMM0N7dn0yaRPcEBunW7JkCXfffTelpaUJec1EGzJkCIWFhZxxxhlcc801jBgxgt69ex9b7+5s3Lix3qGfJ554gs2bN590XVVVFatXr27Vmc+oUaO44YYbmDJlCoMGDWrx60h6OnDgAJs3b+bTTz/ll7/8JZFIhNWrVydsjLxPnz7H/u1cdtllDBw4MCGv296kVXDXOXjwIDNnzuT5559P6Ou21LBhwxgyZAi33norI0aMYPjw4UnZz9GjR1m5ciXRaPSEdZWVldxzzz1UVFTU+/ybbrqJK664gv79+yelPkk/tbW1rFy5ksrKSnbs2MGCBQuora3lww8/POlxejIZGRkUFRVRUlLCjTfeyLnnnpvkqlMvLYMb4NChQ/z2t79l/vz5fPppvfdxSIqePXvSq1cvbr31Vrp37843v/lNBgwYgJmlbOyt7lKihqSyPun46o7Bo0eP8txzz1FdXc2CBQsoLy8nEolQXl7+he379evHoEGDuP3225k6dSpZWVltcoeZ9iBtgxtiB8qKFSu45ppr2LNnT5N/wzdXVlYW2dnZTJ8+nfz8fC688ELOOusscnJyFIQi9XB3qqqqiEajHDhwgEWLFlFdXc3ChQu56qqruPHGGykoKCA7Ozvt/h2ldXBD7ODYt28fc+fO5fHHH0/Y62ZkZNC1a1emTZtGSUkJl156Kb179yYrKyth+xBJN+7O/v376dmzJ5mZTbpJV4fUUHCnxU/FzOjXrx8PPvggvXv3ZvHixXz88cctfr2vfe1rxz7Ay8jIYNiwYWnz3zeRZDMz+vbtm+oy2rW0CO463bp147777uOKK67g9ttv580332zSDVlHjhzJ0KFDmTt3LtnZ2Zx55pl07dq1DSoWETlRWgV3nfPPP5/ly5fzox/9iAceeIBIJPKF9f369eOUU07h1ltvZcCAARQXF3PKKaekqFoRkS9Ky+AGyM7O5s4772TMmDHce++97N27l379+nHTTTdx9tlnM2bMGF1hISLtUqPBbWY5wJ+B7GD7Z939J2b2a+CrwKFg0++4e6nFku6/gUuAiqB9XTKKb62srCyuuuoqpkyZEvuLW2Zp+em1iIRLU864q4BJ7n7EzLKA183sj8G6Oe7+7HHbfwMoCqbxwMPBY7tkZuTk5KS6DBGRJmv0UojgT8MeCRazgqmhawgnA08Gz3sT6GVm+kMXIiIJ0qRr2Mwsw8xKgXLgVXd/K1h1p5mtN7MHzCw7aMsHdsY9fVfQJiIiCdCk4Hb3WncfDQwGxpnZKGAeMAI4F+gDzG3Ojs1slpmtMbM1n3zySTPLFhFJX8361oi7HwRWAhe7++5gOKQKeAIYF2xWBhTEPW1w0Hb8ay1092J3L9YfNBIRabpGg9vM+ptZr2C+C3AhsLlu3Dq4imQKsCF4ylLgGospAQ65++6kVC8ikoaaclVJHrDIzDKIBf0z7v6Sma0ws/6AAaXA94LtXyZ2KeAWYpcDzkh82SIi6avR4Hb39cA5J2mfVM/2DsxufWkiInIy+stIIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZAxd091DZjZYeCDVNeRJP2AT1NdRBJ01H5Bx+2b+hUuQ9y9/8lWZLZ1JfX4wN2LU11EMpjZmo7Yt47aL+i4fVO/Og4NlYiIhIyCW0QkZNpLcC9MdQFJ1FH71lH7BR23b+pXB9EuPpwUEZGmay9n3CIi0kQpD24zu9jMPjCzLWZ2R6rraS4ze9zMys1sQ1xbHzN71cw+DB57B+1mZg8GfV1vZmNSV3nDzKzAzFaa2SYz22hmNwftoe6bmeWY2dtm9m7Qr58G7cPM7K2g/qfNrHPQnh0sbwnWD01l/Y0xswwze8fMXgqWO0q/tpnZe2ZWamZrgrZQH4utkdLgNrMMYAHwDWAkcLWZjUxlTS3wa+Di49ruAJa7exGwPFiGWD+LgmkW8HAb1dgSEeA/3X0kUALMDt6bsPetCpjk7mcDo4GLzawEuAd4wN2/BBwAZgbbzwQOBO0PBNu1ZzcD78ctd5R+AVzg7qPjLv0L+7HYcu6esgk4D1gWtzwPmJfKmlrYj6HAhrjlD4C8YD6P2HXqAP8XuPpk27X3Cfg9cGFH6hvQFVgHjCf2BY7MoP3YcQksA84L5jOD7SzVtdfTn8HEAmwS8BJgHaFfQY3bgH7HtXWYY7G5U6qHSvKBnXHLu4K2sBvo7ruD+T3AwGA+lP0N/ht9DvAWHaBvwXBCKVAOvAp8BBx090iwSXztx/oVrD8E9G3bipvs/wC3A9FguS8do18ADrxiZmvNbFbQFvpjsaXayzcnOyx3dzML7aU7ZtYdeA64xd0/M7Nj68LaN3evBUabWS/gBWBEiktqNTO7FCh397VmNjHV9STBBHcvM7MBwKtmtjl+ZViPxZZK9Rl3GVAQtzw4aAu7vWaWBxA8lgftoeqvmWURC+3fufvzQXOH6BuAux8EVhIbQuhlZnUnMvG1H+tXsL4nsK+NS22KfwUuN7NtwFPEhkv+m/D3CwB3Lwsey4n9sh1HBzoWmyvVwb0aKAo++e4MTAOWprimRFgKXBvMX0tsfLiu/ZrgU+8S4FDcf/XaFYudWv8KeN/d749bFeq+mVn/4EwbM+tCbNz+fWIBPjXY7Ph+1fV3KrDCg4HT9sTd57n7YHcfSuzf0Qp3/3dC3i8AM+tmZrl188DXgQ2E/FhslVQPsgOXAH8nNs74v1JdTwvqXwzsBmqIjaXNJDZWuBz4EPgT0CfY1ohdRfMR8B5QnOr6G+jXBGLjiuuB0mC6JOx9A84C3gn6tQH4cdB+KvA2sAVYAmQH7TnB8pZg/amp7kMT+jgReKmj9Cvow7vBtLEuJ8J+LLZm0jcnRURCJtVDJSIi0kwKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURC5v8Dix/mJUV1WnwAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TjFBWwQP1hVe" + }, + "source": [ + "# Your score" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "GpJpZz3Wbm0X", + "outputId": "7d4677c7-b285-42d3-d8c0-5f0a0d8230c8" + }, + "source": [ + "print(f\"Your final reward is : %.2f\"%np.mean(test_total_reward))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Your final reward is : -147.26\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wUBtYXG2eaqf" + }, + "source": [ + "## Reference\n", + "\n", + "Below are some useful tips for you to get high score.\n", + "\n", + "- [DRL Lecture 1: Policy Gradient (Review)](https://youtu.be/z95ZYgPgXOY)\n", + "- [ML Lecture 23-3: Reinforcement Learning (including Q-learning) start at 30:00](https://youtu.be/2-JNBzCq77c?t=1800)\n", + "- [Lecture 7: Policy Gradient, David Silver](http://www0.cs.ucl.ac.uk/staff/d.silver/web/Teaching_files/pg.pdf)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eZ7VDw-C19Qe" + }, + "source": [ + "" + ] + } + ] +} \ No newline at end of file diff --git a/范例/HW12/HW12_EN.pdf b/范例/HW12/HW12_EN.pdf new file mode 100644 index 0000000..f07f555 Binary files /dev/null and b/范例/HW12/HW12_EN.pdf differ diff --git a/范例/HW12/HW12_ZH.ipynb b/范例/HW12/HW12_ZH.ipynb new file mode 100644 index 0000000..771a9aa --- /dev/null +++ b/范例/HW12/HW12_ZH.ipynb @@ -0,0 +1,3645 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "hw12_reinforcement_learning_chinese_version.ipynb", + "provenance": [], + "collapsed_sections": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "2acab9542fe64b979fa2ac2adb3f10a8": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HBoxModel", + "state": { + "_view_name": "HBoxView", + "_dom_classes": [], + "_model_name": "HBoxModel", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.5.0", + "box_style": "", + "layout": "IPY_MODEL_f288c64b5ff748eb82178bf1de17934f", + "_model_module": "@jupyter-widgets/controls", + "children": [ + "IPY_MODEL_de34e5b178f5470e98e0275102a65042", + "IPY_MODEL_c93cba301cac439ca56fb6b45bd1c4e4" + ] + } + }, + "f288c64b5ff748eb82178bf1de17934f": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "de34e5b178f5470e98e0275102a65042": { + "model_module": "@jupyter-widgets/controls", + "model_name": "FloatProgressModel", + "state": { + "_view_name": "ProgressView", + "style": "IPY_MODEL_43c6ee720b674626ab3a869bda5dd6e3", + "_dom_classes": [], + "description": "Total: -24.0, Final: -40.0: 100%", + "_model_name": "FloatProgressModel", + "bar_style": "success", + "max": 400, + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": 400, + "_view_count": null, + "_view_module_version": "1.5.0", + "orientation": "horizontal", + "min": 0, + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_2465d2b109d34922a486341232d86ad6" + } + }, + "c93cba301cac439ca56fb6b45bd1c4e4": { + "model_module": "@jupyter-widgets/controls", + "model_name": "HTMLModel", + "state": { + "_view_name": "HTMLView", + "style": "IPY_MODEL_aa27187195be4da9874025395eac35eb", + "_dom_classes": [], + "description": "", + "_model_name": "HTMLModel", + "placeholder": "​", + "_view_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "value": " 400/400 [11:02<00:00, 1.66s/it]", + "_view_count": null, + "_view_module_version": "1.5.0", + "description_tooltip": null, + "_model_module": "@jupyter-widgets/controls", + "layout": "IPY_MODEL_02d196d4f9734f998455d92bd9300adb" + } + }, + "43c6ee720b674626ab3a869bda5dd6e3": { + "model_module": "@jupyter-widgets/controls", + "model_name": "ProgressStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "ProgressStyleModel", + "description_width": "initial", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "bar_color": null, + "_model_module": "@jupyter-widgets/controls" + } + }, + "2465d2b109d34922a486341232d86ad6": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + }, + "aa27187195be4da9874025395eac35eb": { + "model_module": "@jupyter-widgets/controls", + "model_name": "DescriptionStyleModel", + "state": { + "_view_name": "StyleView", + "_model_name": "DescriptionStyleModel", + "description_width": "", + "_view_module": "@jupyter-widgets/base", + "_model_module_version": "1.5.0", + "_view_count": null, + "_view_module_version": "1.2.0", + "_model_module": "@jupyter-widgets/controls" + } + }, + "02d196d4f9734f998455d92bd9300adb": { + "model_module": "@jupyter-widgets/base", + "model_name": "LayoutModel", + "state": { + "_view_name": "LayoutView", + "grid_template_rows": null, + "right": null, + "justify_content": null, + "_view_module": "@jupyter-widgets/base", + "overflow": null, + "_model_module_version": "1.2.0", + "_view_count": null, + "flex_flow": null, + "width": null, + "min_width": null, + "border": null, + "align_items": null, + "bottom": null, + "_model_module": "@jupyter-widgets/base", + "top": null, + "grid_column": null, + "overflow_y": null, + "overflow_x": null, + "grid_auto_flow": null, + "grid_area": null, + "grid_template_columns": null, + "flex": null, + "_model_name": "LayoutModel", + "justify_items": null, + "grid_row": null, + "max_height": null, + "align_content": null, + "visibility": null, + "align_self": null, + "height": null, + "min_height": null, + "padding": null, + "grid_auto_rows": null, + "grid_gap": null, + "max_width": null, + "order": null, + "_view_module_version": "1.2.0", + "grid_template_areas": null, + "object_position": null, + "object_fit": null, + "grid_auto_columns": null, + "margin": null, + "display": null, + "left": null + } + } + } + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Fp30SB4bxeQb" + }, + "source": [ + "# **Homework 12 - Reinforcement Learning**\n", + "\n", + "若有任何問題，歡迎來信至助教信箱 ntu-ml-2021spring-ta@googlegroups.com\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yXsnCWPtWSNk" + }, + "source": [ + "## 前置作業\n", + "\n", + "首先我們需要安裝必要的系統套件及 PyPi 套件。\n", + "gym 這個套件由 OpenAI 所提供，是一套用來開發與比較 Reinforcement Learning 演算法的工具包（toolkit）。\n", + "而其餘套件則是為了在 Notebook 中繪圖所需要的套件。" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "5e2bScpnkVbv", + "outputId": "dd8cf053-de15-4a11-c146-5f3405d1e377" + }, + "source": [ + "!apt update\n", + "!apt install python-opengl xvfb -y\n", + "!pip install gym[box2d]==0.18.3 pyvirtualdisplay tqdm numpy==1.19.5 torch==1.8.1" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "\u001b[33m\r0% [Working]\u001b[0m\r \rGet:1 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease [3,626 B]\n", + "\u001b[33m\r0% [Connecting to archive.ubuntu.com (91.189.88.142)] [Waiting for headers] [1 \u001b[0m\u001b[33m\r0% [Connecting to archive.ubuntu.com (91.189.88.142)] [Waiting for headers] [Co\u001b[0m\r \rIgn:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 InRelease\n", + "\u001b[33m\r0% [Connecting to archive.ubuntu.com (91.189.88.142)] [Waiting for headers] [Co\u001b[0m\u001b[33m\r0% [1 InRelease gpgv 3,626 B] [Connecting to archive.ubuntu.com (91.189.88.142)\u001b[0m\r \rIgn:3 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 InRelease\n", + "\u001b[33m\r0% [1 InRelease gpgv 3,626 B] [Connecting to archive.ubuntu.com (91.189.88.142)\u001b[0m\r \rGet:4 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 Release [697 B]\n", + "\u001b[33m\r0% [1 InRelease gpgv 3,626 B] [Connecting to archive.ubuntu.com (91.189.88.142)\u001b[0m\u001b[33m\r0% [1 InRelease gpgv 3,626 B] [Connecting to archive.ubuntu.com (91.189.88.142)\u001b[0m\r \rHit:5 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 Release\n", + "\u001b[33m\r0% [1 InRelease gpgv 3,626 B] [Connecting to archive.ubuntu.com (91.189.88.142)\u001b[0m\r \rGet:6 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 Release.gpg [836 B]\n", + "Get:7 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n", + "Hit:8 http://archive.ubuntu.com/ubuntu bionic InRelease\n", + "Get:9 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease [15.9 kB]\n", + "Get:10 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ Packages [60.9 kB]\n", + "Get:11 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n", + "Hit:13 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n", + "Ign:14 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 Packages\n", + "Get:14 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 Packages [798 kB]\n", + "Get:15 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n", + "Hit:16 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease\n", + "Get:17 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease [21.3 kB]\n", + "Get:18 http://security.ubuntu.com/ubuntu bionic-security/restricted amd64 Packages [423 kB]\n", + "Get:19 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [2,152 kB]\n", + "Get:20 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main Sources [1,770 kB]\n", + "Get:21 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [1,413 kB]\n", + "Get:22 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [2,184 kB]\n", + "Get:23 http://archive.ubuntu.com/ubuntu bionic-updates/restricted amd64 Packages [452 kB]\n", + "Get:24 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [2,584 kB]\n", + "Get:25 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main amd64 Packages [905 kB]\n", + "Get:26 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic/main amd64 Packages [41.5 kB]\n", + "Fetched 13.1 MB in 4s (3,031 kB/s)\n", + "Reading package lists... Done\n", + "Building dependency tree \n", + "Reading state information... Done\n", + "86 packages can be upgraded. Run 'apt list --upgradable' to see them.\n", + "Reading package lists... Done\n", + "Building dependency tree \n", + "Reading state information... Done\n", + "The following package was automatically installed and is no longer required:\n", + " libnvidia-common-460\n", + "Use 'apt autoremove' to remove it.\n", + "Suggested packages:\n", + " libgle3\n", + "The following NEW packages will be installed:\n", + " python-opengl xvfb\n", + "0 upgraded, 2 newly installed, 0 to remove and 86 not upgraded.\n", + "Need to get 1,281 kB of archives.\n", + "After this operation, 7,686 kB of additional disk space will be used.\n", + "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python-opengl all 3.1.0+dfsg-1 [496 kB]\n", + "Get:2 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 xvfb amd64 2:1.19.6-1ubuntu4.9 [784 kB]\n", + "Fetched 1,281 kB in 1s (977 kB/s)\n", + "Selecting previously unselected package python-opengl.\n", + "(Reading database ... 160706 files and directories currently installed.)\n", + "Preparing to unpack .../python-opengl_3.1.0+dfsg-1_all.deb ...\n", + "Unpacking python-opengl (3.1.0+dfsg-1) ...\n", + "Selecting previously unselected package xvfb.\n", + "Preparing to unpack .../xvfb_2%3a1.19.6-1ubuntu4.9_amd64.deb ...\n", + "Unpacking xvfb (2:1.19.6-1ubuntu4.9) ...\n", + "Setting up python-opengl (3.1.0+dfsg-1) ...\n", + "Setting up xvfb (2:1.19.6-1ubuntu4.9) ...\n", + "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n", + "Requirement already satisfied: gym[box2d] in /usr/local/lib/python3.7/dist-packages (0.17.3)\n", + "Collecting pyvirtualdisplay\n", + " Downloading https://files.pythonhosted.org/packages/19/88/7a198a5ee3baa3d547f5a49574cd8c3913b216f5276b690b028f89ffb325/PyVirtualDisplay-2.1-py3-none-any.whl\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (4.41.1)\n", + "Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.3.0)\n", + "Requirement already satisfied: numpy>=1.10.4 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.19.5)\n", + "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.5.0)\n", + "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from gym[box2d]) (1.4.1)\n", + "Collecting box2d-py~=2.3.5; extra == \"box2d\"\n", + "\u001b[?25l Downloading https://files.pythonhosted.org/packages/87/34/da5393985c3ff9a76351df6127c275dcb5749ae0abbe8d5210f06d97405d/box2d_py-2.3.8-cp37-cp37m-manylinux1_x86_64.whl (448kB)\n", + "\u001b[K |████████████████████████████████| 450kB 10.3MB/s \n", + "\u001b[?25hCollecting EasyProcess\n", + " Downloading https://files.pythonhosted.org/packages/48/3c/75573613641c90c6d094059ac28adb748560d99bd27ee6f80cce398f404e/EasyProcess-0.3-py2.py3-none-any.whl\n", + "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym[box2d]) (0.16.0)\n", + "Installing collected packages: EasyProcess, pyvirtualdisplay, box2d-py\n", + "Successfully installed EasyProcess-0.3 box2d-py-2.3.8 pyvirtualdisplay-2.1\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "M_-i3cdoYsks" + }, + "source": [ + "接下來，設置好 virtual display，並引入所有必要的套件。" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "nl2nREINDLiw" + }, + "source": [ + "%%capture\n", + "from pyvirtualdisplay import Display\n", + "virtual_display = Display(visible=0, size=(1400, 900))\n", + "virtual_display.start()\n", + "\n", + "%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from IPython import display\n", + "\n", + "import numpy as np\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.optim as optim\n", + "import torch.nn.functional as F\n", + "from torch.distributions import Categorical\n", + "from tqdm.notebook import tqdm" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HVu9-Vdrl4E3" + }, + "source": [ + "# 請不要更改 random seed !!!!\n", + "# 不然在judgeboi上 你的成績不會被reproduce !!!!" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "fV9i8i2YkRbO" + }, + "source": [ + "seed = 543 # Do not change this\n", + "def fix(env, seed):\n", + " env.seed(seed)\n", + " env.action_space.seed(seed)\n", + " torch.manual_seed(seed)\n", + " torch.cuda.manual_seed(seed)\n", + " torch.cuda.manual_seed_all(seed)\n", + " np.random.seed(seed)\n", + " random.seed(seed)\n", + " torch.set_deterministic(True)\n", + " torch.backends.cudnn.benchmark = False\n", + " torch.backends.cudnn.deterministic = True" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "He0XDx6bzjgC" + }, + "source": [ + "最後，引入 OpenAI 的 gym，並建立一個 [Lunar Lander](https://gym.openai.com/envs/LunarLander-v2/) 環境。" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "N_4-xJcbBt09" + }, + "source": [ + "%%capture\n", + "import gym\n", + "import random\n", + "import numpy as np\n", + "\n", + "env = gym.make('LunarLander-v2')\n", + "\n", + "fix(env, seed)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "NmiAOfqRwRX5" + }, + "source": [ + "import time\n", + "start = time.time()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": { + "id": "LcMjEUWTBEEB", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "7a5146e4-e877-4d26-fd61-652c57ef1f4e" + }, + "source": [ + "!pip freeze" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "absl-py==0.12.0\n", + "alabaster==0.7.12\n", + "albumentations==0.1.12\n", + "altair==4.1.0\n", + "appdirs==1.4.4\n", + "argon2-cffi==20.1.0\n", + "arviz==0.11.2\n", + "astor==0.8.1\n", + "astropy==4.2.1\n", + "astunparse==1.6.3\n", + "async-generator==1.10\n", + "atari-py==0.2.9\n", + "atomicwrites==1.4.0\n", + "attrs==21.2.0\n", + "audioread==2.1.9\n", + "autograd==1.3\n", + "Babel==2.9.1\n", + "backcall==0.2.0\n", + "beautifulsoup4==4.6.3\n", + "bleach==3.3.0\n", + "blis==0.4.1\n", + "bokeh==2.3.2\n", + "Bottleneck==1.3.2\n", + "box2d-py==2.3.8\n", + "branca==0.4.2\n", + "bs4==0.0.1\n", + "CacheControl==0.12.6\n", + "cached-property==1.5.2\n", + "cachetools==4.2.2\n", + "catalogue==1.0.0\n", + "certifi==2020.12.5\n", + "cffi==1.14.5\n", + "cftime==1.5.0\n", + "chainer==7.4.0\n", + "chardet==3.0.4\n", + "click==7.1.2\n", + "cloudpickle==1.3.0\n", + "cmake==3.12.0\n", + "cmdstanpy==0.9.5\n", + "colorcet==2.0.6\n", + "colorlover==0.3.0\n", + "community==1.0.0b1\n", + "contextlib2==0.5.5\n", + "convertdate==2.3.2\n", + "coverage==3.7.1\n", + "coveralls==0.5\n", + "crcmod==1.7\n", + "cufflinks==0.17.3\n", + "cupy-cuda101==7.4.0\n", + "cvxopt==1.2.6\n", + "cvxpy==1.0.31\n", + "cycler==0.10.0\n", + "cymem==2.0.5\n", + "Cython==0.29.23\n", + "daft==0.0.4\n", + "dask==2.12.0\n", + "datascience==0.10.6\n", + "debugpy==1.0.0\n", + "decorator==4.4.2\n", + "defusedxml==0.7.1\n", + "descartes==1.1.0\n", + "dill==0.3.3\n", + "distributed==1.25.3\n", + "dlib==19.18.0\n", + "dm-tree==0.1.6\n", + "docopt==0.6.2\n", + "docutils==0.17.1\n", + "dopamine-rl==1.0.5\n", + "earthengine-api==0.1.266\n", + "easydict==1.9\n", + "EasyProcess==0.3\n", + "ecos==2.0.7.post1\n", + "editdistance==0.5.3\n", + "en-core-web-sm==2.2.5\n", + "entrypoints==0.3\n", + "ephem==3.7.7.1\n", + "et-xmlfile==1.1.0\n", + "fa2==0.3.5\n", + "fastai==1.0.61\n", + "fastdtw==0.3.4\n", + "fastprogress==1.0.0\n", + "fastrlock==0.6\n", + "fbprophet==0.7.1\n", + "feather-format==0.4.1\n", + "filelock==3.0.12\n", + "firebase-admin==4.4.0\n", + "fix-yahoo-finance==0.0.22\n", + "Flask==1.1.4\n", + "flatbuffers==1.12\n", + "folium==0.8.3\n", + "future==0.16.0\n", + "gast==0.4.0\n", + "GDAL==2.2.2\n", + "gdown==3.6.4\n", + "gensim==3.6.0\n", + "geographiclib==1.50\n", + "geopy==1.17.0\n", + "gin-config==0.4.0\n", + "glob2==0.7\n", + "google==2.0.3\n", + "google-api-core==1.26.3\n", + "google-api-python-client==1.12.8\n", + "google-auth==1.30.0\n", + "google-auth-httplib2==0.0.4\n", + "google-auth-oauthlib==0.4.4\n", + "google-cloud-bigquery==1.21.0\n", + "google-cloud-bigquery-storage==1.1.0\n", + "google-cloud-core==1.0.3\n", + "google-cloud-datastore==1.8.0\n", + "google-cloud-firestore==1.7.0\n", + "google-cloud-language==1.2.0\n", + "google-cloud-storage==1.18.1\n", + "google-cloud-translate==1.5.0\n", + "google-colab==1.0.0\n", + "google-pasta==0.2.0\n", + "google-resumable-media==0.4.1\n", + "googleapis-common-protos==1.53.0\n", + "googledrivedownloader==0.4\n", + "graphviz==0.10.1\n", + "greenlet==1.1.0\n", + "grpcio==1.34.1\n", + "gspread==3.0.1\n", + "gspread-dataframe==3.0.8\n", + "gym==0.17.3\n", + "h5py==3.1.0\n", + "HeapDict==1.0.1\n", + "hijri-converter==2.1.1\n", + "holidays==0.10.5.2\n", + "holoviews==1.14.3\n", + "html5lib==1.0.1\n", + "httpimport==0.5.18\n", + "httplib2==0.17.4\n", + "httplib2shim==0.0.3\n", + "humanize==0.5.1\n", + "hyperopt==0.1.2\n", + "ideep4py==2.0.0.post3\n", + "idna==2.10\n", + "imageio==2.4.1\n", + "imagesize==1.2.0\n", + "imbalanced-learn==0.4.3\n", + "imblearn==0.0\n", + "imgaug==0.2.9\n", + "importlib-metadata==4.0.1\n", + "importlib-resources==5.1.3\n", + "imutils==0.5.4\n", + "inflect==2.1.0\n", + "iniconfig==1.1.1\n", + "install==1.3.4\n", + "intel-openmp==2021.2.0\n", + "intervaltree==2.1.0\n", + "ipykernel==4.10.1\n", + "ipython==5.5.0\n", + "ipython-genutils==0.2.0\n", + "ipython-sql==0.3.9\n", + "ipywidgets==7.6.3\n", + "itsdangerous==1.1.0\n", + "jax==0.2.13\n", + "jaxlib==0.1.66+cuda110\n", + "jdcal==1.4.1\n", + "jedi==0.18.0\n", + "jieba==0.42.1\n", + "Jinja2==2.11.3\n", + "joblib==1.0.1\n", + "jpeg4py==0.1.4\n", + "jsonschema==2.6.0\n", + "jupyter==1.0.0\n", + "jupyter-client==5.3.5\n", + "jupyter-console==5.2.0\n", + "jupyter-core==4.7.1\n", + "jupyterlab-pygments==0.1.2\n", + "jupyterlab-widgets==1.0.0\n", + "kaggle==1.5.12\n", + "kapre==0.3.5\n", + "Keras==2.4.3\n", + "keras-nightly==2.5.0.dev2021032900\n", + "Keras-Preprocessing==1.1.2\n", + "keras-vis==0.4.1\n", + "kiwisolver==1.3.1\n", + "korean-lunar-calendar==0.2.1\n", + "librosa==0.8.0\n", + "lightgbm==2.2.3\n", + "llvmlite==0.34.0\n", + "lmdb==0.99\n", + "LunarCalendar==0.0.9\n", + "lxml==4.2.6\n", + "Markdown==3.3.4\n", + "MarkupSafe==2.0.1\n", + "matplotlib==3.2.2\n", + "matplotlib-inline==0.1.2\n", + "matplotlib-venn==0.11.6\n", + "missingno==0.4.2\n", + "mistune==0.8.4\n", + "mizani==0.6.0\n", + "mkl==2019.0\n", + "mlxtend==0.14.0\n", + "more-itertools==8.7.0\n", + "moviepy==0.2.3.5\n", + "mpmath==1.2.1\n", + "msgpack==1.0.2\n", + "multiprocess==0.70.11.1\n", + "multitasking==0.0.9\n", + "murmurhash==1.0.5\n", + "music21==5.5.0\n", + "natsort==5.5.0\n", + "nbclient==0.5.3\n", + "nbconvert==5.6.1\n", + "nbformat==5.1.3\n", + "nest-asyncio==1.5.1\n", + "netCDF4==1.5.6\n", + "networkx==2.5.1\n", + "nibabel==3.0.2\n", + "nltk==3.2.5\n", + "notebook==5.3.1\n", + "numba==0.51.2\n", + "numexpr==2.7.3\n", + "numpy==1.19.5\n", + "nvidia-ml-py3==7.352.0\n", + "oauth2client==4.1.3\n", + "oauthlib==3.1.0\n", + "okgrade==0.4.3\n", + "opencv-contrib-python==4.1.2.30\n", + "opencv-python==4.1.2.30\n", + "openpyxl==2.5.9\n", + "opt-einsum==3.3.0\n", + "osqp==0.6.2.post0\n", + "packaging==20.9\n", + "palettable==3.3.0\n", + "pandas==1.1.5\n", + "pandas-datareader==0.9.0\n", + "pandas-gbq==0.13.3\n", + "pandas-profiling==1.4.1\n", + "pandocfilters==1.4.3\n", + "panel==0.11.3\n", + "param==1.10.1\n", + "parso==0.8.2\n", + "pathlib==1.0.1\n", + "patsy==0.5.1\n", + "pexpect==4.8.0\n", + "pickleshare==0.7.5\n", + "Pillow==7.1.2\n", + "pip-tools==4.5.1\n", + "plac==1.1.3\n", + "plotly==4.4.1\n", + "plotnine==0.6.0\n", + "pluggy==0.7.1\n", + "pooch==1.3.0\n", + "portpicker==1.3.9\n", + "prefetch-generator==1.0.1\n", + "preshed==3.0.5\n", + "prettytable==2.1.0\n", + "progressbar2==3.38.0\n", + "prometheus-client==0.10.1\n", + "promise==2.3\n", + "prompt-toolkit==1.0.18\n", + "protobuf==3.12.4\n", + "psutil==5.4.8\n", + "psycopg2==2.7.6.1\n", + "ptyprocess==0.7.0\n", + "py==1.10.0\n", + "pyarrow==3.0.0\n", + "pyasn1==0.4.8\n", + "pyasn1-modules==0.2.8\n", + "pycocotools==2.0.2\n", + "pycparser==2.20\n", + "pyct==0.4.8\n", + "pydata-google-auth==1.2.0\n", + "pydot==1.3.0\n", + "pydot-ng==2.0.0\n", + "pydotplus==2.0.2\n", + "PyDrive==1.3.1\n", + "pyemd==0.5.1\n", + "pyerfa==2.0.0\n", + "pyglet==1.5.0\n", + "Pygments==2.6.1\n", + "pygobject==3.26.1\n", + "pymc3==3.11.2\n", + "PyMeeus==0.5.11\n", + "pymongo==3.11.4\n", + "pymystem3==0.2.0\n", + "PyOpenGL==3.1.5\n", + "pyparsing==2.4.7\n", + "pyrsistent==0.17.3\n", + "pysndfile==1.3.8\n", + "PySocks==1.7.1\n", + "pystan==2.19.1.1\n", + "pytest==3.6.4\n", + "python-apt==0.0.0\n", + "python-chess==0.23.11\n", + "python-dateutil==2.8.1\n", + "python-louvain==0.15\n", + "python-slugify==5.0.2\n", + "python-utils==2.5.6\n", + "pytz==2018.9\n", + "PyVirtualDisplay==2.1\n", + "pyviz-comms==2.0.1\n", + "PyWavelets==1.1.1\n", + "PyYAML==3.13\n", + "pyzmq==22.0.3\n", + "qdldl==0.1.5.post0\n", + "qtconsole==5.1.0\n", + "QtPy==1.9.0\n", + "regex==2019.12.20\n", + "requests==2.23.0\n", + "requests-oauthlib==1.3.0\n", + "resampy==0.2.2\n", + "retrying==1.3.3\n", + "rpy2==3.4.4\n", + "rsa==4.7.2\n", + "scikit-image==0.16.2\n", + "scikit-learn==0.22.2.post1\n", + "scipy==1.4.1\n", + "screen-resolution-extra==0.0.0\n", + "scs==2.1.3\n", + "seaborn==0.11.1\n", + "semver==2.13.0\n", + "Send2Trash==1.5.0\n", + "setuptools-git==1.2\n", + "Shapely==1.7.1\n", + "simplegeneric==0.8.1\n", + "six==1.15.0\n", + "sklearn==0.0\n", + "sklearn-pandas==1.8.0\n", + "smart-open==5.0.0\n", + "snowballstemmer==2.1.0\n", + "sortedcontainers==2.4.0\n", + "SoundFile==0.10.3.post1\n", + "spacy==2.2.4\n", + "Sphinx==1.8.5\n", + "sphinxcontrib-serializinghtml==1.1.4\n", + "sphinxcontrib-websupport==1.2.4\n", + "SQLAlchemy==1.4.15\n", + "sqlparse==0.4.1\n", + "srsly==1.0.5\n", + "statsmodels==0.10.2\n", + "sympy==1.7.1\n", + "tables==3.4.4\n", + "tabulate==0.8.9\n", + "tblib==1.7.0\n", + "tensorboard==2.5.0\n", + "tensorboard-data-server==0.6.1\n", + "tensorboard-plugin-wit==1.8.0\n", + "tensorflow==2.5.0\n", + "tensorflow-datasets==4.0.1\n", + "tensorflow-estimator==2.5.0\n", + "tensorflow-gcs-config==2.5.0\n", + "tensorflow-hub==0.12.0\n", + "tensorflow-metadata==1.0.0\n", + "tensorflow-probability==0.12.1\n", + "termcolor==1.1.0\n", + "terminado==0.10.0\n", + "testpath==0.5.0\n", + "text-unidecode==1.3\n", + "textblob==0.15.3\n", + "Theano-PyMC==1.1.2\n", + "thinc==7.4.0\n", + "tifffile==2021.4.8\n", + "toml==0.10.2\n", + "toolz==0.11.1\n", + "torch==1.8.1+cu101\n", + "torchsummary==1.5.1\n", + "torchtext==0.9.1\n", + "torchvision==0.9.1+cu101\n", + "tornado==5.1.1\n", + "tqdm==4.41.1\n", + "traitlets==5.0.5\n", + "tweepy==3.10.0\n", + "typeguard==2.7.1\n", + "typing-extensions==3.7.4.3\n", + "tzlocal==1.5.1\n", + "uritemplate==3.0.1\n", + "urllib3==1.24.3\n", + "vega-datasets==0.9.0\n", + "wasabi==0.8.2\n", + "wcwidth==0.2.5\n", + "webencodings==0.5.1\n", + "Werkzeug==1.0.1\n", + "widgetsnbextension==3.5.1\n", + "wordcloud==1.5.0\n", + "wrapt==1.12.1\n", + "xarray==0.18.2\n", + "xgboost==0.90\n", + "xkit==0.0.0\n", + "xlrd==1.1.0\n", + "xlwt==1.3.0\n", + "yellowbrick==0.9.1\n", + "zict==2.0.0\n", + "zipp==3.4.1\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NrkVvTrvWZ5H" + }, + "source": [ + "## 什麼是 Lunar Lander？\n", + "\n", + "“LunarLander-v2” 這個環境是在模擬登月小艇降落在月球表面時的情形。\n", + "這個任務的目標是讓登月小艇「安全地」降落在兩個黃色旗幟間的平地上。\n", + "> Landing pad is always at coordinates (0,0).\n", + "> Coordinates are the first two numbers in state vector.\n", + "\n", + "![](https://gym.openai.com/assets/docs/aeloop-138c89d44114492fd02822303e6b4b07213010bb14ca5856d2d49d6b62d88e53.svg)\n", + "\n", + "所謂的「環境」其實同時包括了 agent 和 environment。\n", + "我們利用 `step()` 這個函式讓 agent 行動，而後函式便會回傳 environment 給予的 observation/state（以下這兩個名詞代表同樣的意思）和 reward。" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bIbp82sljvAt" + }, + "source": [ + "### Observation / State\n", + "\n", + "首先，我們可以看看 environment 回傳給 agent 的 observation 究竟是長什麼樣子的資料：" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "rsXZra3N9R5T", + "outputId": "9512a449-f90a-4545-8aef-dd9aeb9b2b9e" + }, + "source": [ + "print(env.observation_space)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Box(-inf, inf, (8,), float32)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ezdfoThbAQ49" + }, + "source": [ + "`Box(8,)` 說明我們會拿到 8 維的向量作為 observation，其中包含：垂直及水平座標、速度、角度、加速度等等，這部分我們就不細說。\n", + "\n", + "### Action\n", + "\n", + "而在 agent 得到 observation 和 reward 以後，能夠採取的動作有：" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "p1k4dIrBAaKi", + "outputId": "64cd523a-bbff-4569-cae9-f65123b3c604" + }, + "source": [ + "print(env.action_space)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Discrete(4)\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dejXT6PHBrPn" + }, + "source": [ + "`Discrete(4)` 說明 agent 可以採取四種離散的行動：\n", + "- 0 代表不採取任何行動\n", + "- 2 代表主引擎向下噴射\n", + "- 1, 3 則是向左右噴射\n", + "\n", + "接下來，我們嘗試讓 agent 與 environment 互動。\n", + "在進行任何操作前，建議先呼叫 `reset()` 函式讓整個「環境」重置。\n", + "而這個函式同時會回傳「環境」最初始的狀態。" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pi4OmrmZgnWA", + "outputId": "c358ff73-1879-4a74-9579-9ee97740dc16" + }, + "source": [ + "initial_state = env.reset()\n", + "print(initial_state)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "[ 0.00396109 1.4083536 0.40119505 -0.11407257 -0.00458307 -0.09087662\n", + " 0. 0. ]\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uBx0mEqqgxJ9" + }, + "source": [ + "接著，我們試著從 agent 的四種行動空間中，隨機採取一個行動" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "vxkOEXRKgizt", + "outputId": "8912cf80-2310-401b-a37e-c0ded59626ee" + }, + "source": [ + "random_action = env.action_space.sample()\n", + "print(random_action)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "0\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mns-bO01g0-J" + }, + "source": [ + "再利用 `step()` 函式讓 agent 根據我們隨機抽樣出來的 `random_action` 動作。\n", + "而這個函式會回傳四項資訊：\n", + "- observation / state\n", + "- reward\n", + "- 完成與否\n", + "- 其餘資訊" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "E_WViSxGgIk9" + }, + "source": [ + "observation, reward, done, info = env.step(random_action)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "FdieGq7NuBIm" + }, + "source": [ + "第一項資訊 `observation` 即為 agent 採取行動之後，agent 對於環境的 observation 或者說環境的 state 為何。\n", + "而第三項資訊 `done` 則是 `True` 或 `False` 的布林值，當登月小艇成功著陸或是不幸墜毀時，代表這個回合（episode）也就跟著結束了，此時 `step()` 函式便會回傳 `done = True`，而在那之前，`done` 則保持 `False`。" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "yK7r126kuCNp", + "outputId": "3b99114f-e6b4-4a18-c80b-75189083bd55" + }, + "source": [ + "print(done)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "False\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GKdS8vOihxhc" + }, + "source": [ + "### Reward\n", + "\n", + "而「環境」給予的 reward 大致是這樣計算：\n", + "- 小艇墜毀得到 -100 分\n", + "- 小艇在黃旗幟之間成功著地則得 100~140 分\n", + "- 噴射主引擎（向下噴火）每次 -0.3 分\n", + "- 小艇最終完全靜止則再得 100 分\n", + "- 小艇每隻腳碰觸地面 +10 分\n", + "\n", + "> Reward for moving from the top of the screen to landing pad and zero speed is about 100..140 points.\n", + "> If lander moves away from landing pad it loses reward back.\n", + "> Episode finishes if the lander crashes or comes to rest, receiving additional -100 or +100 points.\n", + "> Each leg ground contact is +10.\n", + "> Firing main engine is -0.3 points each frame.\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "vxQNs77hi0_7", + "outputId": "dacd87b3-734e-44f3-c5b4-361b323def84" + }, + "source": [ + "print(reward) # after doing a random action (0), the immediate reward is stored in this " + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "-0.8588900517154912\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Mhqp6D-XgHpe" + }, + "source": [ + "### Random Agent\n", + "\n", + "最後，在進入實做之前，我們就來看看這樣一個 random agent 能否成功登陸月球：" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 269 + }, + "id": "Y3G0bxoccelv", + "outputId": "36096915-445e-40fb-b349-a6a9a5b900d5" + }, + "source": [ + "\n", + "env.reset()\n", + "\n", + "img = plt.imshow(env.render(mode='rgb_array'))\n", + "\n", + "done = False\n", + "while not done:\n", + " action = env.action_space.sample()\n", + " observation, reward, done, _ = env.step(action)\n", + "\n", + " img.set_data(env.render(mode='rgb_array'))\n", + " display.display(plt.gcf())\n", + " display.clear_output(wait=True)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAd2ElEQVR4nO3de3RW9b3n8fc39xCu4RLC3SCtRYvcL8taKD0cKWtmsC0qTr3AaCnWarvmTOfomTVHzzmrx1W0doZlh5baKrQVag9eWBZQBI8VKSAochWIIVxyAgECgYDkxnf+eHbCIwnk9iRPdvJ5rfWs7P3bez/7+wvP82Hn9+z9bHN3REQkPBLiXYCIiDSOgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREKmxYLbzKab2T4zyzWzx1pqPyIiHY21xHncZpYI7AemAUeBD4C73X1PzHcmItLBtNQR93gg193z3L0cWA7MbKF9iYh0KEkt9Lz9gSNR80eBCVdb2cx0+aaIyBXc3epqb6ngrpeZzQPmxWv/IiJh1VLBXQAMjJofELTVcPfFwGLQEbeISGO01Bj3B8AwM7vOzFKA2cDKFtqXiEiH0iJH3O5eaWY/AN4EEoHfuvvultiXiEhH0yKnAza6CA2ViIjUcrUPJ3XlpIhIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREKmWfecNLN84BxQBVS6+1gzywT+CAwB8oE73f1088oUEZFqsTji/pq7j3T3scH8Y8A6dx8GrAvmRUQkRlpiqGQmsCSYXgLc3gL7EBHpsJob3A68ZWbbzGxe0Jbl7oXB9DEgq5n7EBGRKM0a4wa+4u4FZtYHWGtmn0QvdHc3M69rwyDo59W1TERErs7c68zVxj+R2ZNAKfBdYIq7F5pZNvDv7v7FeraNTREiIu2Iu1td7U0eKjGzDDPrUj0N/C2wC1gJ3B+sdj/welP3ISIitTX5iNvMcoBXg9kk4CV3/4mZ9QReBgYBh4icDlhcz3PpiFtE5ApXO+KO2VBJcyi4RURqi/lQiYiIxIeCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiFTb3Cb2W/NrMjMdkW1ZZrZWjM7EPzsEbSbmS00s1wz22Fmo1uyeBGRjqghR9wvAtOvaHsMWOfuw4B1wTzAN4BhwWMesCg2ZYqISLV6g9vd/wIUX9E8E1gSTC8Bbo9qX+oRm4DuZpYdq2JFRKTpY9xZ7l4YTB8DsoLp/sCRqPWOBm21mNk8M9tqZlubWIOISIeU1NwncHc3M2/CdouBxQBN2V5EpKNq6hH38eohkOBnUdBeAAyMWm9A0CYiIjHS1OBeCdwfTN8PvB7Vfl9wdslEoCRqSEVERGLA3K89SmFmy4ApQC/gOPAE8BrwMjAIOATc6e7FZmbAc0TOQrkAzHX3esewNVQiIlKbu1td7fUGd2tQcIuI1Ha14NaVkyIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQmZeoPbzH5rZkVmtiuq7UkzKzCz7cFjRtSyx80s18z2mdltLVW4iEhH1ZCbBX8VKAWWuvtNQduTQKm7P3PFusOBZcB4oB/wNvAFd6+qZx+656SIyBWafM9Jd/8LUNzA/cwElrt7mbsfBHKJhLiIiMRIc8a4f2BmO4KhlB5BW3/gSNQ6R4O2WsxsnpltNbOtzahBRKTDaWpwLwKGAiOBQuBnjX0Cd1/s7mPdfWwTaxAR6ZCaFNzuftzdq9z9EvBrLg+HFAADo1YdELSJiEiMNCm4zSw7avabQPUZJyuB2WaWambXAcOALc0rUUREoiXVt4KZLQOmAL3M7CjwBDDFzEYCDuQD3wNw991m9jKwB6gEHq7vjBIREWmcek8HbJUidDqgiEgtTT4dUERE2hYFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMgpuEZGQUXCLiISMgltEJGQU3CIiIaPgFhEJGQW3iEjI1BvcZjbQzN4xsz1mttvMfhi0Z5rZWjM7EPzsEbSbmS00s1wz22Fmo1u6EyIiHUlDjrgrgb9z9+HAROBhMxsOPAasc/dhwLpgHuAbRO7uPgyYByyKedUiIh1YvcHt7oXu/mEwfQ7YC/QHZgJLgtWWALcH0zOBpR6xCehuZtkxr1xEpINq1Bi3mQ0BRgGbgSx3LwwWHQOygun+wJGozY4GbVc+1zwz22pmWxtZs4hIh9bg4DazzsAK4EfufjZ6mbs74I3Zsbsvdvex7j62MduJiHR0DQpuM0smEtp/cPdXgubj1UMgwc+ioL0AGBi1+YCgTUREYqAhZ5UY8Btgr7s/G7VoJXB/MH0/8HpU+33B2SUTgZKoIRUREWkmi4xyXGMFs68A7wE7gUtB8z8QGed+GRgEHALudPfiIOifA6YDF4C57n7NcWwza9Qwi4hIR+DuVld7vcHdGhTcIiK1XS24deWkiEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQqYhNwseaGbvmNkeM9ttZj8M2p80swIz2x48ZkRt87iZ5ZrZPjO7rSU7ICLS0TTkZsHZQLa7f2hmXYBtwO3AnUCpuz9zxfrDgWXAeKAf8DbwBXevusY+dM9JEZErNPmek+5e6O4fBtPngL1A/2tsMhNY7u5l7n4QyCUS4iIiEgONGuM2syHAKGBz0PQDM9thZr81sx5BW3/gSNRmR7l20IsA8K//+j1++lO46SYYPhz69Yt3Ra1vypQpvPjiF5kxA268EW64ARIT412VtDVJDV3RzDoDK4AfuftZM1sE/Avgwc+fAf+tEc83D5jXuHKlPfvyl3PIzoapUyPzhYWwZ09kes0ayM0Fdzh2DKquOvAWbr1792b8+FJuvDEyX1kJGzdCRQUcPQqvvRZpLymBc+fiV6fEV4OC28ySiYT2H9z9FQB3Px61/NfAG8FsATAwavMBQdvnuPtiYHGwvca4pYYFo3r9+l0+6v7a1yKhXVUFb74Jn30WCfbf/z5+dbak6t9BcjJMnhyZdod77olM79oF+/ZFppcuhePHaz+HtF8NOavEgN8Ae9392aj27KjVvgnsCqZXArPNLNXMrgOGAVtiV7J0RJcuRUK7shIuXIDz5yPh3ZFU/8dVVQUXL0Z+B+fPR3430rE05Ij7FuBeYKeZbQ/a/gG428xGEhkqyQe+B+Duu83sZWAPUAk8fK0zSkSiuUceEBka2B684t58E/LyIsuKi9t/WFX/HiorYf16KC+HggJYuTKyvLS04/3HJZfVG9zuvgGo65SUVdfY5ifAT5pRl3RApaXw5z9Hhj8uXYqM4Z44Ee+qWt/27fDrX8OhQ5Hfw+HD7f8/KmmcBn84KdLSDh+GJ5+MdxXx9+yzsHVrvKuQtkyXvIuIhIyCW0QkZDRUIhITRkJC5EoZ9yrq+yoJkeZQcIvEwLBhX2Xw4LEAHDu2l5KSQgAuXDjDqVMH41matEMKbpFmMjP69x/Bl/rOpFNyJqd7H+R8eeR0mM/KT3Pq7KcAVFSUkZf3PpeCU0SKiw9TWXkxbnVLeCm4RZopK+uL9On2RbqnDSIxIYX05MyaZRVVF7mYeRqAKi9nUN9xuEeC+/ipPZRVnAfg8OGtFBTsbP3iJZQU3CLN1KVLHzql9iYxIaXWsuTENJITL19k3C11UM10/67jqbpURtH5XRQVHWiVWqV9UHCLNENiYgpDhkwgK+PLDVrf7PK1bF1TI1/EcuZifkuUJu2YTgcUaYYePQbSNb0fyYlp8S5FOhAFt8hVdE1KYnT37jXf99A5LY2BPXuSknT5D9WePQfTNa0/SQkKbmk9GioRqUPXpCS+m5PDdRkZ/LmwkLVFRYzOyaFrejonz53jw7w8SEhh0MAx9MkYHu9ypYPREbdIHbLS0hjUqRPJCQnc3L07iWY1p/FVVUUusBk8eCz9e4wmNalbTPfdtWsWSUk6gperU3CL1OFAaSk7S0o4cO4cv87Lo8KNqi4j8B5jOXz2EhVVVaSndyclqQsJFrt7i6WkZDBu9HcYNfLbJCWlxux5pX3RUEkb0L17d1JTUyktLeX8+fPxLkcCvzkYueLRgb59v8SAfrfSK+MLVI3sxYa/LiYxMZkEi91byCyBnJyJDEj7GwqGVFB0Yh+HDulrAqU2BXecJCUlceONNzJz5ky+9a1vMXToUDZs2MD777/PCy+8QGFhYc2f5hIf1d82kpycRs6QSWR3HYmRQEnpfwAwqP8Yene6IXb7c+fUqYN061XAoG6TGHljIefOnaC4+FDM9iHtg4K7lWVkZHDHHXcwceJE7rjjDjIzL19lN336dG677TYeeeQRFi1axM6dO3n11VcV4HE2bNhkhg/+z6Ql9WBXwcvs3r0aALNEzGI52uicOJFH1077MR9Fv+6jGTJkPGfOFHDpUmUM9yNhp+BuJZmZmXzzm9/kxz/+McOGDSMhoe43vJnRp08fnnjiCcrKyti+fTsLFizgwIED7NypS6JbW58+X+BLOdPp0/lGjpXu4NNDGzh79hidO/dusX3u2b+ajKSzDOs5nS8P/RZnzx4nN/cvLbY/CZ96g9vM0oC/AKnB+v/m7k8ENwJeDvQEtgH3unu5maUCS4ExwCngLnfPb6H627SEhATGjBnD448/ztChQxkxYkSjtk9NTWXChAmsWLGCwsJCXnvtNX73u9/xySefcPr06RaqWqolJaUwbNhkBmZO4JJXUnD6Qw4d2gZAnz7XU+kXOXBqNX07jyIxIfJWSkvqQVJC8z5UPHEil/8ov0hm+nX07TyCG4b+DSdP5nHmzNFm90nah4YccZcBU9291MySgQ1mthr478DP3X25mf0SeABYFPw87e7Xm9ls4KfAXS1Uf5vUv39/7rvvPiZNmsS0adNIS2v+qV3Z2dnMnz+f+fPns2nTJn7xi1+wZcsW8vLyqKrSvZhbQq9eOQzMGkPnlL7sKXqND7b+gYsXSwA4dGgb588Xk5CQxNCht9AlNZ1eaal4ykBI7AxARnIW3dOGAJBgCaQkdvncJe8AZZXnOFX6KceP76tpq6i4yN5P1tK3z5cY0e87DOn9VU584QAfbF2mIRMBGnazYAdKg9nk4OHAVOC/Bu1LgCeJBPfMYBrg34DnzMy8nX+zfHp6OuPHj+fuu+9m2rRpDBky5KrDIU1V/aafNGkSEyZMoLy8nGXLlvH0009z7NgxHYXHUEZGL0aNmMWArhM4eX4fBw9v4MyZgprlVVXlNWFbWLiHvunpjOmZSb5155xFjri7d+9PVtYXAUhOTKdP18iFOobRJ+MmkhJSKas6x2flZ7hwofhz+z9//hRbP1pORlof+mR8idTULiQmpii4BWjgGLeZJRIZDrke+AXwKXDG3atfRUeB/sF0f+AIgLtXmlkJkeGUkzGsu00wM3r16sV3v/tdxowZw4wZM2JydN0QCQkJpKWlMWfOHO666y4OHTrEc889x5IlSygvL6eioqJV6oiVtLQ0Nm7cSEZGRhu5e8xFjp3YTueuXbhYdpa8g/9OWloKUPsbAAHOAu8UFwOXA7i4eD/5+ZGx6ZSUTgwYMBKIvG4GDxpHcnI6AGlpyWRkZFBVVUl+fj4lJSV06tSJM2cOs333y/ToMZBDh7dSUXGhJTssIWKNeZOYWXfgVeB/Ay+6+/VB+0BgtbvfZGa7gOnufjRY9ikwwd1PXvFc84B5weyYZvekFSUkJJCVlcWcOXN49NFH6dOnT8yPrpuioqKCU6dOsXbtWn71q1/x17/+tc2dkVL9e+ratSuzZs2qmf/+979PVlZWPEurxSyRlJR03J3y8tieX5+S0qnmjBT3S5SX1w7lV155hY8//pjExBSqqsp57bXXOHny8tuorf3bSuy5u9XV3qjgBjCzfwQ+A/4e6BscVU8CnnT328zszWD6r2aWBBwDel9rqCQ9Pd0vXmz7dwJJTExk2rRpPPbYYwwZMoTBgwfHu6SrKi4upqioiOeee44//elPFBUVxaWOTp068fWvf52EhATMjEcffZTs7GySk5PJycmpNeYrdXN3Dh06RPX75OLFiyxYsIALFy4H/rZt2zh6VB9gtidNDm4z6w1UuPsZM0sH3iLygeP9wIqoDyd3uPv/M7OHgS+7+/zgw8lvufud19rHzTff7IsWLaqZLy4u5umnn6750O3MmTPs3r27wZ2NtRtuuIEHH3yQW265hdGjR5OSUvefy23Vnj17yM/P56mnnmLTpk1UVsZ+nDQ1NZUxY8ZgZowePZrZs2fXtI8aNapN/EXS3u3fv7/miNzdWbhwIQUFl8flCwoKyM/Pj1N10hTNCe4RRD58TCTy3SYvu/s/m1kOkdMBM4GPgHvcvSw4ffB3wCgiA36z3T3vWvsYO3asb9169Ut7i4qK2LRpU818ZWUlzzzzDGfOnKlpi/WHc926dWPq1KnMmTOHcePGkZ2dXf9GbVxFRQVvv/02W7du5fnnn+fw4cONfo5+/frRrVvkS5VGjhxZE9CdOnVi6tSpCug2LDc3lz179tTML1u2jI8//rhmPi8vj7KysniUJlcRs6GSllBfcF/J3WuN723atOlzL8oXXniBgwcv3137s88+o6Sk5JrPm5iYSO/evZk8eTIPPfQQt956a7sMInenoKCA1atX88orr/Dee+/V+o6Ubt26kZ4e+fBsxIgRzJo1C4ApU6aQk5MDRD5ka4+/n47i0qVLn/sg+PXXX+fUqVMArFmzho0bNzbofSMtp10Fd0NcvHjxc+c37969m9WrV9fMf/bZZyxevJjy8nIgcp70I488wty5c0lNTQ3dcEhTlZWVsX79erZs2fK59hkzZjB8eOT0taSkJFJT9U11HUn1mUl79uxh1apV/PGPf+Tw4cOUlZW1yFCb1K3DBXd9Ll26xMmTJ2uO3JOTk8nMzNSHZSJ1KC4upry8nNWrV7N582YOHDjAu+++q4u/WpiCW0RipqSkhGPHjrF48WLy8vLYsWMHeXnX/ChLmkDBLSIt5uDBgxw5coQFCxZQUlLCjh07OHv2bLzLCj0Ft4i0mo0bN7Jv3z6effZZKisryc3N1dh4Eyi4RaRVVZ/9VVVVxYoVK/joo4/4/e9/T2VlJSdOnIh3eaGg4BaRuKqqqqKsrIzTp0/z4osv8v777/Pee+9RWVlJGK6cjgcFt4i0KefPn6e0tJT9+/fz0ksvsXbtWvLz82udX96RXS24dQccEYmLjIwMMjIyyMrK4tZbb6WgoIDS0lLWr1/PW2+9xbp16zh37ly8y2yTdMQtIm2Ou7Njxw5+9rOfsWrVqporOjuaqx1x63plEWlzzIybb76ZpUuXsnr1aubOnUvv3i13n8+wUXCLSJs2btw4nn/+edasWcNDDz1EZmZmvEuKOwW3iLR5CQkJjB49moULF/Luu+8yf/58OnXqFO+y4kbBLSKhkZSUxE033cTChQv54IMPmDt3bqvdLrAtUXCLSOgkJyczfPhwFi9ezLZt27j33nvp3LlzvMtqNQpuEQmtpKQkhg8fzpIlS9iwYQP33ntvhxgDV3CLSOhFn4WyZs0aXnrpJUaMGEFiYmK8S2sRCm4RaVfGjRvH7Nmz2bJlC0uXLmXEiBHt7sYo9Qa3maWZ2RYz+9jMdpvZPwXtL5rZQTPbHjxGBu1mZgvNLNfMdpjZ6JbuhIhINDMjNTWVu+++m82bN/PLX/6SUaNGkZycHO/SYqIhR9xlwFR3vxkYCUw3s4nBsh+7+8jgsT1o+wYwLHjMAxbVekYRkVZgZqSlpTF37lzeeecdFi1axPjx40N/r9R6q/eI0mA2OXhc6zr5mcDSYLtNQHczC/8t0kUk1Lp168YDDzzAqlWrWLJkCZMmTYp3SU3WoP92zCzRzLYDRcBad98cLPpJMBzyczOrvptsf+BI1OZHgzYRkbjr2bMn99xzD2+88QbLly9n3Lhx8S6pRo8ePZg8eTKTJ0++5umNDfp2QHevAkaaWXfgVTO7CXgcOAakAIuBvwf+uaEFmtk8IkMpDBo0qKGbiYjERGZmJnfddRczZsxg1apVPPXUU+zatSumN0DOyMggJyenVvv111/Pgw8+WKu9V69ejB8/HoCxY8de9Xkb9bWu7n7GzN4Bprv7M0FzmZm9APyPYL4AGBi12YCg7crnWkwk8Bk7dmz8v6JQRDqkLl26cOedd3L77bezYsUKFixYwN69eykvL6+1bmpqap1fdnXdddfxwAMP1Grv27cv06ZNq9VuZpjV+cV/DVJvcJtZb6AiCO10YBrwUzPLdvdCi+z9dmBXsMlK4AdmthyYAJS4e2GTKxQRaWHRZ6F8+9vfZtmyZRw7dqzWegMGDGDWrFm12hMSElr1lMOGHHFnA0vMLJHImPjL7v6Gma0PQt2A7cD8YP1VwAwgF7gAzI192SIisVcd4HPmzIl3KddUb3C7+w5gVB3tU6+yvgMPN780ERGpS7hPZhQR6YAU3CIiIaPgFhEJGQW3iEjIKLhFREJGwS0iEjIKbhGRkFFwi4iEjIJbRCRkFNwiIiGj4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZBTcIiIho+AWEQkZBbeISMgouEVEQkbBLSISMubu8a4BMzsH7It3HS2kF3Ay3kW0gPbaL2i/fVO/wmWwu/eua0FSa1dyFfvcfWy8i2gJZra1PfatvfYL2m/f1K/2Q0MlIiIho+AWEQmZthLci+NdQAtqr31rr/2C9ts39audaBMfToqISMO1lSNuERFpoLgHt5lNN7N9ZpZrZo/Fu57GMrPfmlmRme2Kass0s7VmdiD42SNoNzNbGPR1h5mNjl/l12ZmA83sHTPbY2a7zeyHQXuo+2ZmaWa2xcw+Dvr1T0H7dWa2Oaj/j2aWErSnBvO5wfIh8ay/PmaWaGYfmdkbwXx76Ve+me00s+1mtjVoC/VrsTniGtxmlgj8AvgGMBy428yGx7OmJngRmH5F22PAOncfBqwL5iHSz2HBYx6wqJVqbIpK4O/cfTgwEXg4+LcJe9/KgKnufjMwEphuZhOBnwI/d/frgdPAA8H6DwCng/afB+u1ZT8E9kbNt5d+AXzN3UdGnfoX9tdi07l73B7AJODNqPnHgcfjWVMT+zEE2BU1vw/IDqaziZynDvAr4O661mvrD+B1YFp76hvQCfgQmEDkAo6koL3mdQm8CUwKppOC9SzetV+lPwOIBNhU4A3A2kO/ghrzgV5XtLWb12JjH/EeKukPHImaPxq0hV2WuxcG08eArGA6lP0N/oweBWymHfQtGE7YDhQBa4FPgTPuXhmsEl17Tb+C5SVAz9atuMH+D/A/gUvBfE/aR78AHHjLzLaZ2bygLfSvxaZqK1dOtlvu7mYW2lN3zKwzsAL4kbufNbOaZWHtm7tXASPNrDvwKnBDnEtqNjP7T0CRu28zsynxrqcFfMXdC8ysD7DWzD6JXhjW12JTxfuIuwAYGDU/IGgLu+Nmlg0Q/CwK2kPVXzNLJhLaf3D3V4LmdtE3AHc/A7xDZAihu5lVH8hE117Tr2B5N+BUK5faELcA/8XM8oHlRIZL/i/h7xcA7l4Q/Cwi8p/teNrRa7Gx4h3cHwDDgk++U4DZwMo41xQLK4H7g+n7iYwPV7ffF3zqPREoifpTr02xyKH1b4C97v5s1KJQ983MegdH2phZOpFx+71EAnxWsNqV/aru7yxgvQcDp22Juz/u7gPcfQiR99F6d/8OIe8XgJllmFmX6mngb4FdhPy12CzxHmQHZgD7iYwz/q9419OE+pcBhUAFkbG0B4iMFa4DDgBvA5nBukbkLJpPgZ3A2HjXf41+fYXIuOIOYHvwmBH2vgEjgI+Cfu0C/jFozwG2ALnAn4DUoD0tmM8NlufEuw8N6OMU4I320q+gDx8Hj93VORH212JzHrpyUkQkZOI9VCIiIo2k4BYRCRkFt4hIyCi4RURCRsEtIhIyCm4RkZBRcIuIhIyCW0QkZP4/aOAjA9p4Wi8AAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "F5paWqo7tWL2" + }, + "source": [ + "## Policy Gradient\n", + "\n", + "現在來搭建一個簡單的 policy network。\n", + "我們預設模型的輸入是 8-dim 的 observation，輸出則是離散的四個動作之一：" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "J8tdmeD-tZew" + }, + "source": [ + "class PolicyGradientNetwork(nn.Module):\n", + "\n", + " def __init__(self):\n", + " super().__init__()\n", + " self.fc1 = nn.Linear(8, 16)\n", + " self.fc2 = nn.Linear(16, 16)\n", + " self.fc3 = nn.Linear(16, 4)\n", + "\n", + " def forward(self, state):\n", + " hid = torch.tanh(self.fc1(state))\n", + " hid = torch.tanh(self.fc2(hid))\n", + " return F.softmax(self.fc3(hid), dim=-1)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ynbqJrhIFTC3" + }, + "source": [ + "再來，搭建一個簡單的 agent，並搭配上方的 policy network 來採取行動。\n", + "這個 agent 能做到以下幾件事：\n", + "- `learn()`：從記下來的 log probabilities 及 rewards 來更新 policy network。\n", + "- `sample()`：從 environment 得到 observation 之後，利用 policy network 得出應該採取的行動。\n", + "而此函式除了回傳抽樣出來的 action，也會回傳此次抽樣的 log probabilities。" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "zZo-IxJx286z" + }, + "source": [ + "\n", + "class PolicyGradientAgent():\n", + " \n", + " def __init__(self, network):\n", + " self.network = network\n", + " self.optimizer = optim.SGD(self.network.parameters(), lr=0.001)\n", + " \n", + " def forward(self, state):\n", + " return self.network(state)\n", + " def learn(self, log_probs, rewards):\n", + " loss = (-log_probs * rewards).sum() # You don't need to revise this to pass simple baseline (but you can)\n", + "\n", + " self.optimizer.zero_grad()\n", + " loss.backward()\n", + " self.optimizer.step()\n", + " \n", + " def sample(self, state):\n", + " action_prob = self.network(torch.FloatTensor(state))\n", + " action_dist = Categorical(action_prob)\n", + " action = action_dist.sample()\n", + " log_prob = action_dist.log_prob(action)\n", + " return action.item(), log_prob\n", + "\n", + " def save(self, PATH): # You should not revise this\n", + " Agent_Dict = {\n", + " \"network\" : self.network.state_dict(),\n", + " \"optimizer\" : self.optimizer.state_dict()\n", + " }\n", + " torch.save(Agent_Dict, PATH)\n", + "\n", + " def load(self, PATH): # You should not revise this\n", + " checkpoint = torch.load(PATH)\n", + " self.network.load_state_dict(checkpoint[\"network\"])\n", + " #如果要儲存過程或是中斷訓練後想繼續可以用喔 ^_^\n", + " self.optimizer.load_state_dict(checkpoint[\"optimizer\"])\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ehPlnTKyRZf9" + }, + "source": [ + "最後，建立一個 network 和 agent，就可以開始進行訓練了。" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "GfJIvML-RYjL" + }, + "source": [ + "network = PolicyGradientNetwork()\n", + "agent = PolicyGradientAgent(network)\n", + "#agent = PolicyGradientAgent()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ouv23glgf5Qt" + }, + "source": [ + "## 訓練 Agent\n", + "\n", + "現在我們開始訓練 agent。\n", + "透過讓 agent 和 environment 互動，我們記住每一組對應的 log probabilities 及 reward，並在成功登陸或者不幸墜毀後，回放這些「記憶」來訓練 policy network。" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000, + "referenced_widgets": [ + "2acab9542fe64b979fa2ac2adb3f10a8", + "f288c64b5ff748eb82178bf1de17934f", + "de34e5b178f5470e98e0275102a65042", + "c93cba301cac439ca56fb6b45bd1c4e4", + "43c6ee720b674626ab3a869bda5dd6e3", + "2465d2b109d34922a486341232d86ad6", + "aa27187195be4da9874025395eac35eb", + "02d196d4f9734f998455d92bd9300adb" + ] + }, + "id": "vg5rxBBaf38_", + "outputId": "eae0c9f4-0efc-40fe-a29e-7f7194613f6d" + }, + "source": [ + "agent.network.train() # 訓練前，先確保 network 處在 training 模式\n", + "EPISODE_PER_BATCH = 5 # 每蒐集 5 個 episodes 更新一次 agent\n", + "NUM_BATCH = 400 # 總共更新 400 次\n", + "\n", + "avg_total_rewards, avg_final_rewards = [], []\n", + "\n", + "prg_bar = tqdm(range(NUM_BATCH))\n", + "for batch in prg_bar:\n", + "\n", + " log_probs, rewards = [], []\n", + " total_rewards, final_rewards = [], []\n", + "\n", + " # 蒐集訓練資料\n", + " for episode in range(EPISODE_PER_BATCH):\n", + " \n", + " state = env.reset()\n", + " total_reward, total_step = 0, 0\n", + " seq_rewards = []\n", + " while True:\n", + "\n", + " action, log_prob = agent.sample(state) # at , log(at|st)\n", + " next_state, reward, done, _ = env.step(action)\n", + "\n", + " log_probs.append(log_prob) # [log(a1|s1), log(a2|s2), ...., log(at|st)]\n", + " # seq_rewards.append(reward)\n", + " state = next_state\n", + " total_reward += reward\n", + " total_step += 1\n", + " rewards.append(reward) #改這裡\n", + " # ! 重要 ！\n", + " # 現在的reward 的implementation 為每個時刻的瞬時reward, 給定action_list : a1, a2, a3 ......\n", + " # reward : r1, r2 ,r3 ......\n", + " # medium：將reward調整成accumulative decaying reward, 給定action_list : a1, a2, a3 ......\n", + " # reward : r1+0.99*r2+0.99^2*r3+......, r2+0.99*r3+0.99^2*r4+...... ,r3+0.99*r4+0.99^2*r5+ ......\n", + " # boss : implement DQN\n", + " if done:\n", + " final_rewards.append(reward)\n", + " total_rewards.append(total_reward)\n", + " break\n", + "\n", + " print(f\"rewards looks like \", np.shape(rewards)) \n", + " print(f\"log_probs looks like \", np.shape(log_probs)) \n", + " # 紀錄訓練過程\n", + " avg_total_reward = sum(total_rewards) / len(total_rewards)\n", + " avg_final_reward = sum(final_rewards) / len(final_rewards)\n", + " avg_total_rewards.append(avg_total_reward)\n", + " avg_final_rewards.append(avg_final_reward)\n", + " prg_bar.set_description(f\"Total: {avg_total_reward: 4.1f}, Final: {avg_final_reward: 4.1f}\")\n", + "\n", + " # 更新網路\n", + " # rewards = np.concatenate(rewards, axis=0)\n", + " rewards = (rewards - np.mean(rewards)) / (np.std(rewards) + 1e-9) # 將 reward 正規標準化\n", + " agent.learn(torch.stack(log_probs), torch.from_numpy(rewards))\n", + " print(\"logs prob looks like \", torch.stack(log_probs).size())\n", + " print(\"torch.from_numpy(rewards) looks like \", torch.from_numpy(rewards).size())" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2acab9542fe64b979fa2ac2adb3f10a8", + "version_minor": 0, + "version_major": 2 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=400.0), HTML(value='')))" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "stream", + "text": [ + "rewards looks like (448,)\n", + "log_probs looks like (448,)\n", + "logs prob looks like torch.Size([448])\n", + "torch.from_numpy(rewards) looks like torch.Size([448])\n", + "rewards looks like (515,)\n", + "log_probs looks like (515,)\n", + "logs prob looks like torch.Size([515])\n", + "torch.from_numpy(rewards) looks like torch.Size([515])\n", + "rewards looks like (392,)\n", + "log_probs looks like (392,)\n", + "logs prob looks like torch.Size([392])\n", + "torch.from_numpy(rewards) looks like torch.Size([392])\n", + "rewards looks like (518,)\n", + "log_probs looks like (518,)\n", + "logs prob looks like torch.Size([518])\n", + "torch.from_numpy(rewards) looks like torch.Size([518])\n", + "rewards looks like (472,)\n", + "log_probs looks like (472,)\n", + "logs prob looks like torch.Size([472])\n", + "torch.from_numpy(rewards) looks like torch.Size([472])\n", + "rewards looks like (530,)\n", + "log_probs looks like (530,)\n", + "logs prob looks like torch.Size([530])\n", + "torch.from_numpy(rewards) looks like torch.Size([530])\n", + "rewards looks like (463,)\n", + "log_probs looks like (463,)\n", + "logs prob looks like torch.Size([463])\n", + "torch.from_numpy(rewards) looks like torch.Size([463])\n", + "rewards looks like (540,)\n", + "log_probs looks like (540,)\n", + "logs prob looks like torch.Size([540])\n", + "torch.from_numpy(rewards) looks like torch.Size([540])\n", + "rewards looks like (513,)\n", + "log_probs looks like (513,)\n", + "logs prob looks like torch.Size([513])\n", + "torch.from_numpy(rewards) looks like torch.Size([513])\n", + "rewards looks like (449,)\n", + "log_probs looks like (449,)\n", + "logs prob looks like torch.Size([449])\n", + "torch.from_numpy(rewards) looks like torch.Size([449])\n", + "rewards looks like (602,)\n", + "log_probs looks like (602,)\n", + "logs prob looks like torch.Size([602])\n", + "torch.from_numpy(rewards) looks like torch.Size([602])\n", + "rewards looks like (542,)\n", + "log_probs looks like (542,)\n", + "logs prob looks like torch.Size([542])\n", + "torch.from_numpy(rewards) looks like torch.Size([542])\n", + "rewards looks like (503,)\n", + "log_probs looks like (503,)\n", + "logs prob looks like torch.Size([503])\n", + "torch.from_numpy(rewards) looks like torch.Size([503])\n", + "rewards looks like (470,)\n", + "log_probs looks like (470,)\n", + "logs prob looks like torch.Size([470])\n", + "torch.from_numpy(rewards) looks like torch.Size([470])\n", + "rewards looks like (518,)\n", + "log_probs looks like (518,)\n", + "logs prob looks like torch.Size([518])\n", + "torch.from_numpy(rewards) looks like torch.Size([518])\n", + "rewards looks like (421,)\n", + "log_probs looks like (421,)\n", + "logs prob looks like torch.Size([421])\n", + "torch.from_numpy(rewards) looks like torch.Size([421])\n", + "rewards looks like (592,)\n", + "log_probs looks like (592,)\n", + "logs prob looks like torch.Size([592])\n", + "torch.from_numpy(rewards) looks like torch.Size([592])\n", + "rewards looks like (520,)\n", + "log_probs looks like (520,)\n", + "logs prob looks like torch.Size([520])\n", + "torch.from_numpy(rewards) looks like torch.Size([520])\n", + "rewards looks like (494,)\n", + "log_probs looks like (494,)\n", + "logs prob looks like torch.Size([494])\n", + "torch.from_numpy(rewards) looks like torch.Size([494])\n", + "rewards looks like (461,)\n", + "log_probs looks like (461,)\n", + "logs prob looks like torch.Size([461])\n", + "torch.from_numpy(rewards) looks like torch.Size([461])\n", + "rewards looks like (572,)\n", + "log_probs looks like (572,)\n", + "logs prob looks like torch.Size([572])\n", + "torch.from_numpy(rewards) looks like torch.Size([572])\n", + "rewards looks like (593,)\n", + "log_probs looks like (593,)\n", + "logs prob looks like torch.Size([593])\n", + "torch.from_numpy(rewards) looks like torch.Size([593])\n", + "rewards looks like (569,)\n", + "log_probs looks like (569,)\n", + "logs prob looks like torch.Size([569])\n", + "torch.from_numpy(rewards) looks like torch.Size([569])\n", + "rewards looks like (546,)\n", + "log_probs looks like (546,)\n", + "logs prob looks like torch.Size([546])\n", + "torch.from_numpy(rewards) looks like torch.Size([546])\n", + "rewards looks like (612,)\n", + "log_probs looks like (612,)\n", + "logs prob looks like torch.Size([612])\n", + "torch.from_numpy(rewards) looks like torch.Size([612])\n", + "rewards looks like (534,)\n", + "log_probs looks like (534,)\n", + "logs prob looks like torch.Size([534])\n", + "torch.from_numpy(rewards) looks like torch.Size([534])\n", + "rewards looks like (513,)\n", + "log_probs looks like (513,)\n", + "logs prob looks like torch.Size([513])\n", + "torch.from_numpy(rewards) looks like torch.Size([513])\n", + "rewards looks like (513,)\n", + "log_probs looks like (513,)\n", + "logs prob looks like torch.Size([513])\n", + "torch.from_numpy(rewards) looks like torch.Size([513])\n", + "rewards looks like (535,)\n", + "log_probs looks like (535,)\n", + "logs prob looks like torch.Size([535])\n", + "torch.from_numpy(rewards) looks like torch.Size([535])\n", + "rewards looks like (533,)\n", + "log_probs looks like (533,)\n", + "logs prob looks like torch.Size([533])\n", + "torch.from_numpy(rewards) looks like torch.Size([533])\n", + "rewards looks like (521,)\n", + "log_probs looks like (521,)\n", + "logs prob looks like torch.Size([521])\n", + "torch.from_numpy(rewards) looks like torch.Size([521])\n", + "rewards looks like (566,)\n", + "log_probs looks like (566,)\n", + "logs prob looks like torch.Size([566])\n", + "torch.from_numpy(rewards) looks like torch.Size([566])\n", + "rewards looks like (586,)\n", + "log_probs looks like (586,)\n", + "logs prob looks like torch.Size([586])\n", + "torch.from_numpy(rewards) looks like torch.Size([586])\n", + "rewards looks like (575,)\n", + "log_probs looks like (575,)\n", + "logs prob looks like torch.Size([575])\n", + "torch.from_numpy(rewards) looks like torch.Size([575])\n", + "rewards looks like (709,)\n", + "log_probs looks like (709,)\n", + "logs prob looks like torch.Size([709])\n", + "torch.from_numpy(rewards) looks like torch.Size([709])\n", + "rewards looks like (486,)\n", + "log_probs looks like (486,)\n", + "logs prob looks like torch.Size([486])\n", + "torch.from_numpy(rewards) looks like torch.Size([486])\n", + "rewards looks like (557,)\n", + "log_probs looks like (557,)\n", + "logs prob looks like torch.Size([557])\n", + "torch.from_numpy(rewards) looks like torch.Size([557])\n", + "rewards looks like (517,)\n", + "log_probs looks like (517,)\n", + "logs prob looks like torch.Size([517])\n", + "torch.from_numpy(rewards) looks like torch.Size([517])\n", + "rewards looks like (550,)\n", + "log_probs looks like (550,)\n", + "logs prob looks like torch.Size([550])\n", + "torch.from_numpy(rewards) looks like torch.Size([550])\n", + "rewards looks like (690,)\n", + "log_probs looks like (690,)\n", + "logs prob looks like torch.Size([690])\n", + "torch.from_numpy(rewards) looks like torch.Size([690])\n", + "rewards looks like (591,)\n", + "log_probs looks like (591,)\n", + "logs prob looks like torch.Size([591])\n", + "torch.from_numpy(rewards) looks like torch.Size([591])\n", + "rewards looks like (689,)\n", + "log_probs looks like (689,)\n", + "logs prob looks like torch.Size([689])\n", + "torch.from_numpy(rewards) looks like torch.Size([689])\n", + "rewards looks like (1059,)\n", + "log_probs looks like (1059,)\n", + "logs prob looks like torch.Size([1059])\n", + "torch.from_numpy(rewards) looks like torch.Size([1059])\n", + "rewards looks like (619,)\n", + "log_probs looks like (619,)\n", + "logs prob looks like torch.Size([619])\n", + "torch.from_numpy(rewards) looks like torch.Size([619])\n", + "rewards looks like (527,)\n", + "log_probs looks like (527,)\n", + "logs prob looks like torch.Size([527])\n", + "torch.from_numpy(rewards) looks like torch.Size([527])\n", + "rewards looks like (514,)\n", + "log_probs looks like (514,)\n", + "logs prob looks like torch.Size([514])\n", + "torch.from_numpy(rewards) looks like torch.Size([514])\n", + "rewards looks like (655,)\n", + "log_probs looks like (655,)\n", + "logs prob looks like torch.Size([655])\n", + "torch.from_numpy(rewards) looks like torch.Size([655])\n", + "rewards looks like (667,)\n", + "log_probs looks like (667,)\n", + "logs prob looks like torch.Size([667])\n", + "torch.from_numpy(rewards) looks like torch.Size([667])\n", + "rewards looks like (712,)\n", + "log_probs looks like (712,)\n", + "logs prob looks like torch.Size([712])\n", + "torch.from_numpy(rewards) looks like torch.Size([712])\n", + "rewards looks like (636,)\n", + "log_probs looks like (636,)\n", + "logs prob looks like torch.Size([636])\n", + "torch.from_numpy(rewards) looks like torch.Size([636])\n", + "rewards looks like (620,)\n", + "log_probs looks like (620,)\n", + "logs prob looks like torch.Size([620])\n", + "torch.from_numpy(rewards) looks like torch.Size([620])\n", + "rewards looks like (543,)\n", + "log_probs looks like (543,)\n", + "logs prob looks like torch.Size([543])\n", + "torch.from_numpy(rewards) looks like torch.Size([543])\n", + "rewards looks like (586,)\n", + "log_probs looks like (586,)\n", + "logs prob looks like torch.Size([586])\n", + "torch.from_numpy(rewards) looks like torch.Size([586])\n", + "rewards looks like (498,)\n", + "log_probs looks like (498,)\n", + "logs prob looks like torch.Size([498])\n", + "torch.from_numpy(rewards) looks like torch.Size([498])\n", + "rewards looks like (586,)\n", + "log_probs looks like (586,)\n", + "logs prob looks like torch.Size([586])\n", + "torch.from_numpy(rewards) looks like torch.Size([586])\n", + "rewards looks like (591,)\n", + "log_probs looks like (591,)\n", + "logs prob looks like torch.Size([591])\n", + "torch.from_numpy(rewards) looks like torch.Size([591])\n", + "rewards looks like (693,)\n", + "log_probs looks like (693,)\n", + "logs prob looks like torch.Size([693])\n", + "torch.from_numpy(rewards) looks like torch.Size([693])\n", + "rewards looks like (648,)\n", + "log_probs looks like (648,)\n", + "logs prob looks like torch.Size([648])\n", + "torch.from_numpy(rewards) looks like torch.Size([648])\n", + "rewards looks like (513,)\n", + "log_probs looks like (513,)\n", + "logs prob looks like torch.Size([513])\n", + "torch.from_numpy(rewards) looks like torch.Size([513])\n", + "rewards looks like (574,)\n", + "log_probs looks like (574,)\n", + "logs prob looks like torch.Size([574])\n", + "torch.from_numpy(rewards) looks like torch.Size([574])\n", + "rewards looks like (718,)\n", + "log_probs looks like (718,)\n", + "logs prob looks like torch.Size([718])\n", + "torch.from_numpy(rewards) looks like torch.Size([718])\n", + "rewards looks like (730,)\n", + "log_probs looks like (730,)\n", + "logs prob looks like torch.Size([730])\n", + "torch.from_numpy(rewards) looks like torch.Size([730])\n", + "rewards looks like (668,)\n", + "log_probs looks like (668,)\n", + "logs prob looks like torch.Size([668])\n", + "torch.from_numpy(rewards) looks like torch.Size([668])\n", + "rewards looks like (754,)\n", + "log_probs looks like (754,)\n", + "logs prob looks like torch.Size([754])\n", + "torch.from_numpy(rewards) looks like torch.Size([754])\n", + "rewards looks like (712,)\n", + "log_probs looks like (712,)\n", + "logs prob looks like torch.Size([712])\n", + "torch.from_numpy(rewards) looks like torch.Size([712])\n", + "rewards looks like (470,)\n", + "log_probs looks like (470,)\n", + "logs prob looks like torch.Size([470])\n", + "torch.from_numpy(rewards) looks like torch.Size([470])\n", + "rewards looks like (665,)\n", + "log_probs looks like (665,)\n", + "logs prob looks like torch.Size([665])\n", + "torch.from_numpy(rewards) looks like torch.Size([665])\n", + "rewards looks like (585,)\n", + "log_probs looks like (585,)\n", + "logs prob looks like torch.Size([585])\n", + "torch.from_numpy(rewards) looks like torch.Size([585])\n", + "rewards looks like (512,)\n", + "log_probs looks like (512,)\n", + "logs prob looks like torch.Size([512])\n", + "torch.from_numpy(rewards) looks like torch.Size([512])\n", + "rewards looks like (702,)\n", + "log_probs looks like (702,)\n", + "logs prob looks like torch.Size([702])\n", + "torch.from_numpy(rewards) looks like torch.Size([702])\n", + "rewards looks like (596,)\n", + "log_probs looks like (596,)\n", + "logs prob looks like torch.Size([596])\n", + "torch.from_numpy(rewards) looks like torch.Size([596])\n", + "rewards looks like (626,)\n", + "log_probs looks like (626,)\n", + "logs prob looks like torch.Size([626])\n", + "torch.from_numpy(rewards) looks like torch.Size([626])\n", + "rewards looks like (566,)\n", + "log_probs looks like (566,)\n", + "logs prob looks like torch.Size([566])\n", + "torch.from_numpy(rewards) looks like torch.Size([566])\n", + "rewards looks like (717,)\n", + "log_probs looks like (717,)\n", + "logs prob looks like torch.Size([717])\n", + "torch.from_numpy(rewards) looks like torch.Size([717])\n", + "rewards looks like (708,)\n", + "log_probs looks like (708,)\n", + "logs prob looks like torch.Size([708])\n", + "torch.from_numpy(rewards) looks like torch.Size([708])\n", + "rewards looks like (565,)\n", + "log_probs looks like (565,)\n", + "logs prob looks like torch.Size([565])\n", + "torch.from_numpy(rewards) looks like torch.Size([565])\n", + "rewards looks like (450,)\n", + "log_probs looks like (450,)\n", + "logs prob looks like torch.Size([450])\n", + "torch.from_numpy(rewards) looks like torch.Size([450])\n", + "rewards looks like (584,)\n", + "log_probs looks like (584,)\n", + "logs prob looks like torch.Size([584])\n", + "torch.from_numpy(rewards) looks like torch.Size([584])\n", + "rewards looks like (670,)\n", + "log_probs looks like (670,)\n", + "logs prob looks like torch.Size([670])\n", + "torch.from_numpy(rewards) looks like torch.Size([670])\n", + "rewards looks like (691,)\n", + "log_probs looks like (691,)\n", + "logs prob looks like torch.Size([691])\n", + "torch.from_numpy(rewards) looks like torch.Size([691])\n", + "rewards looks like (760,)\n", + "log_probs looks like (760,)\n", + "logs prob looks like torch.Size([760])\n", + "torch.from_numpy(rewards) looks like torch.Size([760])\n", + "rewards looks like (752,)\n", + "log_probs looks like (752,)\n", + "logs prob looks like torch.Size([752])\n", + "torch.from_numpy(rewards) looks like torch.Size([752])\n", + "rewards looks like (478,)\n", + "log_probs looks like (478,)\n", + "logs prob looks like torch.Size([478])\n", + "torch.from_numpy(rewards) looks like torch.Size([478])\n", + "rewards looks like (553,)\n", + "log_probs looks like (553,)\n", + "logs prob looks like torch.Size([553])\n", + "torch.from_numpy(rewards) looks like torch.Size([553])\n", + "rewards looks like (1660,)\n", + "log_probs looks like (1660,)\n", + "logs prob looks like torch.Size([1660])\n", + "torch.from_numpy(rewards) looks like torch.Size([1660])\n", + "rewards looks like (751,)\n", + "log_probs looks like (751,)\n", + "logs prob looks like torch.Size([751])\n", + "torch.from_numpy(rewards) looks like torch.Size([751])\n", + "rewards looks like (801,)\n", + "log_probs looks like (801,)\n", + "logs prob looks like torch.Size([801])\n", + "torch.from_numpy(rewards) looks like torch.Size([801])\n", + "rewards looks like (715,)\n", + "log_probs looks like (715,)\n", + "logs prob looks like torch.Size([715])\n", + "torch.from_numpy(rewards) looks like torch.Size([715])\n", + "rewards looks like (708,)\n", + "log_probs looks like (708,)\n", + "logs prob looks like torch.Size([708])\n", + "torch.from_numpy(rewards) looks like torch.Size([708])\n", + "rewards looks like (609,)\n", + "log_probs looks like (609,)\n", + "logs prob looks like torch.Size([609])\n", + "torch.from_numpy(rewards) looks like torch.Size([609])\n", + "rewards looks like (732,)\n", + "log_probs looks like (732,)\n", + "logs prob looks like torch.Size([732])\n", + "torch.from_numpy(rewards) looks like torch.Size([732])\n", + "rewards looks like (603,)\n", + "log_probs looks like (603,)\n", + "logs prob looks like torch.Size([603])\n", + "torch.from_numpy(rewards) looks like torch.Size([603])\n", + "rewards looks like (603,)\n", + "log_probs looks like (603,)\n", + "logs prob looks like torch.Size([603])\n", + "torch.from_numpy(rewards) looks like torch.Size([603])\n", + "rewards looks like (665,)\n", + "log_probs looks like (665,)\n", + "logs prob looks like torch.Size([665])\n", + "torch.from_numpy(rewards) looks like torch.Size([665])\n", + "rewards looks like (658,)\n", + "log_probs looks like (658,)\n", + "logs prob looks like torch.Size([658])\n", + "torch.from_numpy(rewards) looks like torch.Size([658])\n", + "rewards looks like (783,)\n", + "log_probs looks like (783,)\n", + "logs prob looks like torch.Size([783])\n", + "torch.from_numpy(rewards) looks like torch.Size([783])\n", + "rewards looks like (652,)\n", + "log_probs looks like (652,)\n", + "logs prob looks like torch.Size([652])\n", + "torch.from_numpy(rewards) looks like torch.Size([652])\n", + "rewards looks like (892,)\n", + "log_probs looks like (892,)\n", + "logs prob looks like torch.Size([892])\n", + "torch.from_numpy(rewards) looks like torch.Size([892])\n", + "rewards looks like (821,)\n", + "log_probs looks like (821,)\n", + "logs prob looks like torch.Size([821])\n", + "torch.from_numpy(rewards) looks like torch.Size([821])\n", + "rewards looks like (986,)\n", + "log_probs looks like (986,)\n", + "logs prob looks like torch.Size([986])\n", + "torch.from_numpy(rewards) looks like torch.Size([986])\n", + "rewards looks like (916,)\n", + "log_probs looks like (916,)\n", + "logs prob looks like torch.Size([916])\n", + "torch.from_numpy(rewards) looks like torch.Size([916])\n", + "rewards looks like (742,)\n", + "log_probs looks like (742,)\n", + "logs prob looks like torch.Size([742])\n", + "torch.from_numpy(rewards) looks like torch.Size([742])\n", + "rewards looks like (604,)\n", + "log_probs looks like (604,)\n", + "logs prob looks like torch.Size([604])\n", + "torch.from_numpy(rewards) looks like torch.Size([604])\n", + "rewards looks like (818,)\n", + "log_probs looks like (818,)\n", + "logs prob looks like torch.Size([818])\n", + "torch.from_numpy(rewards) looks like torch.Size([818])\n", + "rewards looks like (855,)\n", + "log_probs looks like (855,)\n", + "logs prob looks like torch.Size([855])\n", + "torch.from_numpy(rewards) looks like torch.Size([855])\n", + "rewards looks like (795,)\n", + "log_probs looks like (795,)\n", + "logs prob looks like torch.Size([795])\n", + "torch.from_numpy(rewards) looks like torch.Size([795])\n", + "rewards looks like (868,)\n", + "log_probs looks like (868,)\n", + "logs prob looks like torch.Size([868])\n", + "torch.from_numpy(rewards) looks like torch.Size([868])\n", + "rewards looks like (800,)\n", + "log_probs looks like (800,)\n", + "logs prob looks like torch.Size([800])\n", + "torch.from_numpy(rewards) looks like torch.Size([800])\n", + "rewards looks like (820,)\n", + "log_probs looks like (820,)\n", + "logs prob looks like torch.Size([820])\n", + "torch.from_numpy(rewards) looks like torch.Size([820])\n", + "rewards looks like (760,)\n", + "log_probs looks like (760,)\n", + "logs prob looks like torch.Size([760])\n", + "torch.from_numpy(rewards) looks like torch.Size([760])\n", + "rewards looks like (886,)\n", + "log_probs looks like (886,)\n", + "logs prob looks like torch.Size([886])\n", + "torch.from_numpy(rewards) looks like torch.Size([886])\n", + "rewards looks like (1027,)\n", + "log_probs looks like (1027,)\n", + "logs prob looks like torch.Size([1027])\n", + "torch.from_numpy(rewards) looks like torch.Size([1027])\n", + "rewards looks like (819,)\n", + "log_probs looks like (819,)\n", + "logs prob looks like torch.Size([819])\n", + "torch.from_numpy(rewards) looks like torch.Size([819])\n", + "rewards looks like (934,)\n", + "log_probs looks like (934,)\n", + "logs prob looks like torch.Size([934])\n", + "torch.from_numpy(rewards) looks like torch.Size([934])\n", + "rewards looks like (1648,)\n", + "log_probs looks like (1648,)\n", + "logs prob looks like torch.Size([1648])\n", + "torch.from_numpy(rewards) looks like torch.Size([1648])\n", + "rewards looks like (1057,)\n", + "log_probs looks like (1057,)\n", + "logs prob looks like torch.Size([1057])\n", + "torch.from_numpy(rewards) looks like torch.Size([1057])\n", + "rewards looks like (861,)\n", + "log_probs looks like (861,)\n", + "logs prob looks like torch.Size([861])\n", + "torch.from_numpy(rewards) looks like torch.Size([861])\n", + "rewards looks like (1533,)\n", + "log_probs looks like (1533,)\n", + "logs prob looks like torch.Size([1533])\n", + "torch.from_numpy(rewards) looks like torch.Size([1533])\n", + "rewards looks like (920,)\n", + "log_probs looks like (920,)\n", + "logs prob looks like torch.Size([920])\n", + "torch.from_numpy(rewards) looks like torch.Size([920])\n", + "rewards looks like (905,)\n", + "log_probs looks like (905,)\n", + "logs prob looks like torch.Size([905])\n", + "torch.from_numpy(rewards) looks like torch.Size([905])\n", + "rewards looks like (814,)\n", + "log_probs looks like (814,)\n", + "logs prob looks like torch.Size([814])\n", + "torch.from_numpy(rewards) looks like torch.Size([814])\n", + "rewards looks like (809,)\n", + "log_probs looks like (809,)\n", + "logs prob looks like torch.Size([809])\n", + "torch.from_numpy(rewards) looks like torch.Size([809])\n", + "rewards looks like (873,)\n", + "log_probs looks like (873,)\n", + "logs prob looks like torch.Size([873])\n", + "torch.from_numpy(rewards) looks like torch.Size([873])\n", + "rewards looks like (727,)\n", + "log_probs looks like (727,)\n", + "logs prob looks like torch.Size([727])\n", + "torch.from_numpy(rewards) looks like torch.Size([727])\n", + "rewards looks like (1129,)\n", + "log_probs looks like (1129,)\n", + "logs prob looks like torch.Size([1129])\n", + "torch.from_numpy(rewards) looks like torch.Size([1129])\n", + "rewards looks like (1394,)\n", + "log_probs looks like (1394,)\n", + "logs prob looks like torch.Size([1394])\n", + "torch.from_numpy(rewards) looks like torch.Size([1394])\n", + "rewards looks like (884,)\n", + "log_probs looks like (884,)\n", + "logs prob looks like torch.Size([884])\n", + "torch.from_numpy(rewards) looks like torch.Size([884])\n", + "rewards looks like (1132,)\n", + "log_probs looks like (1132,)\n", + "logs prob looks like torch.Size([1132])\n", + "torch.from_numpy(rewards) looks like torch.Size([1132])\n", + "rewards looks like (1007,)\n", + "log_probs looks like (1007,)\n", + "logs prob looks like torch.Size([1007])\n", + "torch.from_numpy(rewards) looks like torch.Size([1007])\n", + "rewards looks like (711,)\n", + "log_probs looks like (711,)\n", + "logs prob looks like torch.Size([711])\n", + "torch.from_numpy(rewards) looks like torch.Size([711])\n", + "rewards looks like (836,)\n", + "log_probs looks like (836,)\n", + "logs prob looks like torch.Size([836])\n", + "torch.from_numpy(rewards) looks like torch.Size([836])\n", + "rewards looks like (1514,)\n", + "log_probs looks like (1514,)\n", + "logs prob looks like torch.Size([1514])\n", + "torch.from_numpy(rewards) looks like torch.Size([1514])\n", + "rewards looks like (896,)\n", + "log_probs looks like (896,)\n", + "logs prob looks like torch.Size([896])\n", + "torch.from_numpy(rewards) looks like torch.Size([896])\n", + "rewards looks like (912,)\n", + "log_probs looks like (912,)\n", + "logs prob looks like torch.Size([912])\n", + "torch.from_numpy(rewards) looks like torch.Size([912])\n", + "rewards looks like (1478,)\n", + "log_probs looks like (1478,)\n", + "logs prob looks like torch.Size([1478])\n", + "torch.from_numpy(rewards) looks like torch.Size([1478])\n", + "rewards looks like (1279,)\n", + "log_probs looks like (1279,)\n", + "logs prob looks like torch.Size([1279])\n", + "torch.from_numpy(rewards) looks like torch.Size([1279])\n", + "rewards looks like (676,)\n", + "log_probs looks like (676,)\n", + "logs prob looks like torch.Size([676])\n", + "torch.from_numpy(rewards) looks like torch.Size([676])\n", + "rewards looks like (1768,)\n", + "log_probs looks like (1768,)\n", + "logs prob looks like torch.Size([1768])\n", + "torch.from_numpy(rewards) looks like torch.Size([1768])\n", + "rewards looks like (897,)\n", + "log_probs looks like (897,)\n", + "logs prob looks like torch.Size([897])\n", + "torch.from_numpy(rewards) looks like torch.Size([897])\n", + "rewards looks like (1252,)\n", + "log_probs looks like (1252,)\n", + "logs prob looks like torch.Size([1252])\n", + "torch.from_numpy(rewards) looks like torch.Size([1252])\n", + "rewards looks like (995,)\n", + "log_probs looks like (995,)\n", + "logs prob looks like torch.Size([995])\n", + "torch.from_numpy(rewards) looks like torch.Size([995])\n", + "rewards looks like (1075,)\n", + "log_probs looks like (1075,)\n", + "logs prob looks like torch.Size([1075])\n", + "torch.from_numpy(rewards) looks like torch.Size([1075])\n", + "rewards looks like (878,)\n", + "log_probs looks like (878,)\n", + "logs prob looks like torch.Size([878])\n", + "torch.from_numpy(rewards) looks like torch.Size([878])\n", + "rewards looks like (1341,)\n", + "log_probs looks like (1341,)\n", + "logs prob looks like torch.Size([1341])\n", + "torch.from_numpy(rewards) looks like torch.Size([1341])\n", + "rewards looks like (1518,)\n", + "log_probs looks like (1518,)\n", + "logs prob looks like torch.Size([1518])\n", + "torch.from_numpy(rewards) looks like torch.Size([1518])\n", + "rewards looks like (1781,)\n", + "log_probs looks like (1781,)\n", + "logs prob looks like torch.Size([1781])\n", + "torch.from_numpy(rewards) looks like torch.Size([1781])\n", + "rewards looks like (1725,)\n", + "log_probs looks like (1725,)\n", + "logs prob looks like torch.Size([1725])\n", + "torch.from_numpy(rewards) looks like torch.Size([1725])\n", + "rewards looks like (1602,)\n", + "log_probs looks like (1602,)\n", + "logs prob looks like torch.Size([1602])\n", + "torch.from_numpy(rewards) looks like torch.Size([1602])\n", + "rewards looks like (846,)\n", + "log_probs looks like (846,)\n", + "logs prob looks like torch.Size([846])\n", + "torch.from_numpy(rewards) looks like torch.Size([846])\n", + "rewards looks like (1211,)\n", + "log_probs looks like (1211,)\n", + "logs prob looks like torch.Size([1211])\n", + "torch.from_numpy(rewards) looks like torch.Size([1211])\n", + "rewards looks like (3273,)\n", + "log_probs looks like (3273,)\n", + "logs prob looks like torch.Size([3273])\n", + "torch.from_numpy(rewards) looks like torch.Size([3273])\n", + "rewards looks like (744,)\n", + "log_probs looks like (744,)\n", + "logs prob looks like torch.Size([744])\n", + "torch.from_numpy(rewards) looks like torch.Size([744])\n", + "rewards looks like (1751,)\n", + "log_probs looks like (1751,)\n", + "logs prob looks like torch.Size([1751])\n", + "torch.from_numpy(rewards) looks like torch.Size([1751])\n", + "rewards looks like (1244,)\n", + "log_probs looks like (1244,)\n", + "logs prob looks like torch.Size([1244])\n", + "torch.from_numpy(rewards) looks like torch.Size([1244])\n", + "rewards looks like (1313,)\n", + "log_probs looks like (1313,)\n", + "logs prob looks like torch.Size([1313])\n", + "torch.from_numpy(rewards) looks like torch.Size([1313])\n", + "rewards looks like (1993,)\n", + "log_probs looks like (1993,)\n", + "logs prob looks like torch.Size([1993])\n", + "torch.from_numpy(rewards) looks like torch.Size([1993])\n", + "rewards looks like (709,)\n", + "log_probs looks like (709,)\n", + "logs prob looks like torch.Size([709])\n", + "torch.from_numpy(rewards) looks like torch.Size([709])\n", + "rewards looks like (934,)\n", + "log_probs looks like (934,)\n", + "logs prob looks like torch.Size([934])\n", + "torch.from_numpy(rewards) looks like torch.Size([934])\n", + "rewards looks like (1386,)\n", + "log_probs looks like (1386,)\n", + "logs prob looks like torch.Size([1386])\n", + "torch.from_numpy(rewards) looks like torch.Size([1386])\n", + "rewards looks like (635,)\n", + "log_probs looks like (635,)\n", + "logs prob looks like torch.Size([635])\n", + "torch.from_numpy(rewards) looks like torch.Size([635])\n", + "rewards looks like (750,)\n", + "log_probs looks like (750,)\n", + "logs prob looks like torch.Size([750])\n", + "torch.from_numpy(rewards) looks like torch.Size([750])\n", + "rewards looks like (1832,)\n", + "log_probs looks like (1832,)\n", + "logs prob looks like torch.Size([1832])\n", + "torch.from_numpy(rewards) looks like torch.Size([1832])\n", + "rewards looks like (1237,)\n", + "log_probs looks like (1237,)\n", + "logs prob looks like torch.Size([1237])\n", + "torch.from_numpy(rewards) looks like torch.Size([1237])\n", + "rewards looks like (1605,)\n", + "log_probs looks like (1605,)\n", + "logs prob looks like torch.Size([1605])\n", + "torch.from_numpy(rewards) looks like torch.Size([1605])\n", + "rewards looks like (718,)\n", + "log_probs looks like (718,)\n", + "logs prob looks like torch.Size([718])\n", + "torch.from_numpy(rewards) looks like torch.Size([718])\n", + "rewards looks like (966,)\n", + "log_probs looks like (966,)\n", + "logs prob looks like torch.Size([966])\n", + "torch.from_numpy(rewards) looks like torch.Size([966])\n", + "rewards looks like (2696,)\n", + "log_probs looks like (2696,)\n", + "logs prob looks like torch.Size([2696])\n", + "torch.from_numpy(rewards) looks like torch.Size([2696])\n", + "rewards looks like (762,)\n", + "log_probs looks like (762,)\n", + "logs prob looks like torch.Size([762])\n", + "torch.from_numpy(rewards) looks like torch.Size([762])\n", + "rewards looks like (1048,)\n", + "log_probs looks like (1048,)\n", + "logs prob looks like torch.Size([1048])\n", + "torch.from_numpy(rewards) looks like torch.Size([1048])\n", + "rewards looks like (1573,)\n", + "log_probs looks like (1573,)\n", + "logs prob looks like torch.Size([1573])\n", + "torch.from_numpy(rewards) looks like torch.Size([1573])\n", + "rewards looks like (2192,)\n", + "log_probs looks like (2192,)\n", + "logs prob looks like torch.Size([2192])\n", + "torch.from_numpy(rewards) looks like torch.Size([2192])\n", + "rewards looks like (599,)\n", + "log_probs looks like (599,)\n", + "logs prob looks like torch.Size([599])\n", + "torch.from_numpy(rewards) looks like torch.Size([599])\n", + "rewards looks like (758,)\n", + "log_probs looks like (758,)\n", + "logs prob looks like torch.Size([758])\n", + "torch.from_numpy(rewards) looks like torch.Size([758])\n", + "rewards looks like (1955,)\n", + "log_probs looks like (1955,)\n", + "logs prob looks like torch.Size([1955])\n", + "torch.from_numpy(rewards) looks like torch.Size([1955])\n", + "rewards looks like (1770,)\n", + "log_probs looks like (1770,)\n", + "logs prob looks like torch.Size([1770])\n", + "torch.from_numpy(rewards) looks like torch.Size([1770])\n", + "rewards looks like (1343,)\n", + "log_probs looks like (1343,)\n", + "logs prob looks like torch.Size([1343])\n", + "torch.from_numpy(rewards) looks like torch.Size([1343])\n", + "rewards looks like (507,)\n", + "log_probs looks like (507,)\n", + "logs prob looks like torch.Size([507])\n", + "torch.from_numpy(rewards) looks like torch.Size([507])\n", + "rewards looks like (1343,)\n", + "log_probs looks like (1343,)\n", + "logs prob looks like torch.Size([1343])\n", + "torch.from_numpy(rewards) looks like torch.Size([1343])\n", + "rewards looks like (1341,)\n", + "log_probs looks like (1341,)\n", + "logs prob looks like torch.Size([1341])\n", + "torch.from_numpy(rewards) looks like torch.Size([1341])\n", + "rewards looks like (1489,)\n", + "log_probs looks like (1489,)\n", + "logs prob looks like torch.Size([1489])\n", + "torch.from_numpy(rewards) looks like torch.Size([1489])\n", + "rewards looks like (3342,)\n", + "log_probs looks like (3342,)\n", + "logs prob looks like torch.Size([3342])\n", + "torch.from_numpy(rewards) looks like torch.Size([3342])\n", + "rewards looks like (1891,)\n", + "log_probs looks like (1891,)\n", + "logs prob looks like torch.Size([1891])\n", + "torch.from_numpy(rewards) looks like torch.Size([1891])\n", + "rewards looks like (1401,)\n", + "log_probs looks like (1401,)\n", + "logs prob looks like torch.Size([1401])\n", + "torch.from_numpy(rewards) looks like torch.Size([1401])\n", + "rewards looks like (2964,)\n", + "log_probs looks like (2964,)\n", + "logs prob looks like torch.Size([2964])\n", + "torch.from_numpy(rewards) looks like torch.Size([2964])\n", + "rewards looks like (1404,)\n", + "log_probs looks like (1404,)\n", + "logs prob looks like torch.Size([1404])\n", + "torch.from_numpy(rewards) looks like torch.Size([1404])\n", + "rewards looks like (780,)\n", + "log_probs looks like (780,)\n", + "logs prob looks like torch.Size([780])\n", + "torch.from_numpy(rewards) looks like torch.Size([780])\n", + "rewards looks like (1632,)\n", + "log_probs looks like (1632,)\n", + "logs prob looks like torch.Size([1632])\n", + "torch.from_numpy(rewards) looks like torch.Size([1632])\n", + "rewards looks like (1578,)\n", + "log_probs looks like (1578,)\n", + "logs prob looks like torch.Size([1578])\n", + "torch.from_numpy(rewards) looks like torch.Size([1578])\n", + "rewards looks like (1082,)\n", + "log_probs looks like (1082,)\n", + "logs prob looks like torch.Size([1082])\n", + "torch.from_numpy(rewards) looks like torch.Size([1082])\n", + "rewards looks like (1423,)\n", + "log_probs looks like (1423,)\n", + "logs prob looks like torch.Size([1423])\n", + "torch.from_numpy(rewards) looks like torch.Size([1423])\n", + "rewards looks like (2867,)\n", + "log_probs looks like (2867,)\n", + "logs prob looks like torch.Size([2867])\n", + "torch.from_numpy(rewards) looks like torch.Size([2867])\n", + "rewards looks like (1733,)\n", + "log_probs looks like (1733,)\n", + "logs prob looks like torch.Size([1733])\n", + "torch.from_numpy(rewards) looks like torch.Size([1733])\n", + "rewards looks like (646,)\n", + "log_probs looks like (646,)\n", + "logs prob looks like torch.Size([646])\n", + "torch.from_numpy(rewards) looks like torch.Size([646])\n", + "rewards looks like (1576,)\n", + "log_probs looks like (1576,)\n", + "logs prob looks like torch.Size([1576])\n", + "torch.from_numpy(rewards) looks like torch.Size([1576])\n", + "rewards looks like (1869,)\n", + "log_probs looks like (1869,)\n", + "logs prob looks like torch.Size([1869])\n", + "torch.from_numpy(rewards) looks like torch.Size([1869])\n", + "rewards looks like (1862,)\n", + "log_probs looks like (1862,)\n", + "logs prob looks like torch.Size([1862])\n", + "torch.from_numpy(rewards) looks like torch.Size([1862])\n", + "rewards looks like (3182,)\n", + "log_probs looks like (3182,)\n", + "logs prob looks like torch.Size([3182])\n", + "torch.from_numpy(rewards) looks like torch.Size([3182])\n", + "rewards looks like (1746,)\n", + "log_probs looks like (1746,)\n", + "logs prob looks like torch.Size([1746])\n", + "torch.from_numpy(rewards) looks like torch.Size([1746])\n", + "rewards looks like (1855,)\n", + "log_probs looks like (1855,)\n", + "logs prob looks like torch.Size([1855])\n", + "torch.from_numpy(rewards) looks like torch.Size([1855])\n", + "rewards looks like (2710,)\n", + "log_probs looks like (2710,)\n", + "logs prob looks like torch.Size([2710])\n", + "torch.from_numpy(rewards) looks like torch.Size([2710])\n", + "rewards looks like (1707,)\n", + "log_probs looks like (1707,)\n", + "logs prob looks like torch.Size([1707])\n", + "torch.from_numpy(rewards) looks like torch.Size([1707])\n", + "rewards looks like (1723,)\n", + "log_probs looks like (1723,)\n", + "logs prob looks like torch.Size([1723])\n", + "torch.from_numpy(rewards) looks like torch.Size([1723])\n", + "rewards looks like (1590,)\n", + "log_probs looks like (1590,)\n", + "logs prob looks like torch.Size([1590])\n", + "torch.from_numpy(rewards) looks like torch.Size([1590])\n", + "rewards looks like (1432,)\n", + "log_probs looks like (1432,)\n", + "logs prob looks like torch.Size([1432])\n", + "torch.from_numpy(rewards) looks like torch.Size([1432])\n", + "rewards looks like (2742,)\n", + "log_probs looks like (2742,)\n", + "logs prob looks like torch.Size([2742])\n", + "torch.from_numpy(rewards) looks like torch.Size([2742])\n", + "rewards looks like (3007,)\n", + "log_probs looks like (3007,)\n", + "logs prob looks like torch.Size([3007])\n", + "torch.from_numpy(rewards) looks like torch.Size([3007])\n", + "rewards looks like (2064,)\n", + "log_probs looks like (2064,)\n", + "logs prob looks like torch.Size([2064])\n", + "torch.from_numpy(rewards) looks like torch.Size([2064])\n", + "rewards looks like (1447,)\n", + "log_probs looks like (1447,)\n", + "logs prob looks like torch.Size([1447])\n", + "torch.from_numpy(rewards) looks like torch.Size([1447])\n", + "rewards looks like (4007,)\n", + "log_probs looks like (4007,)\n", + "logs prob looks like torch.Size([4007])\n", + "torch.from_numpy(rewards) looks like torch.Size([4007])\n", + "rewards looks like (611,)\n", + "log_probs looks like (611,)\n", + "logs prob looks like torch.Size([611])\n", + "torch.from_numpy(rewards) looks like torch.Size([611])\n", + "rewards looks like (1633,)\n", + "log_probs looks like (1633,)\n", + "logs prob looks like torch.Size([1633])\n", + "torch.from_numpy(rewards) looks like torch.Size([1633])\n", + "rewards looks like (3295,)\n", + "log_probs looks like (3295,)\n", + "logs prob looks like torch.Size([3295])\n", + "torch.from_numpy(rewards) looks like torch.Size([3295])\n", + "rewards looks like (975,)\n", + "log_probs looks like (975,)\n", + "logs prob looks like torch.Size([975])\n", + "torch.from_numpy(rewards) looks like torch.Size([975])\n", + "rewards looks like (1991,)\n", + "log_probs looks like (1991,)\n", + "logs prob looks like torch.Size([1991])\n", + "torch.from_numpy(rewards) looks like torch.Size([1991])\n", + "rewards looks like (2409,)\n", + "log_probs looks like (2409,)\n", + "logs prob looks like torch.Size([2409])\n", + "torch.from_numpy(rewards) looks like torch.Size([2409])\n", + "rewards looks like (1587,)\n", + "log_probs looks like (1587,)\n", + "logs prob looks like torch.Size([1587])\n", + "torch.from_numpy(rewards) looks like torch.Size([1587])\n", + "rewards looks like (1334,)\n", + "log_probs looks like (1334,)\n", + "logs prob looks like torch.Size([1334])\n", + "torch.from_numpy(rewards) looks like torch.Size([1334])\n", + "rewards looks like (1070,)\n", + "log_probs looks like (1070,)\n", + "logs prob looks like torch.Size([1070])\n", + "torch.from_numpy(rewards) looks like torch.Size([1070])\n", + "rewards looks like (1082,)\n", + "log_probs looks like (1082,)\n", + "logs prob looks like torch.Size([1082])\n", + "torch.from_numpy(rewards) looks like torch.Size([1082])\n", + "rewards looks like (1084,)\n", + "log_probs looks like (1084,)\n", + "logs prob looks like torch.Size([1084])\n", + "torch.from_numpy(rewards) looks like torch.Size([1084])\n", + "rewards looks like (1192,)\n", + "log_probs looks like (1192,)\n", + "logs prob looks like torch.Size([1192])\n", + "torch.from_numpy(rewards) looks like torch.Size([1192])\n", + "rewards looks like (1287,)\n", + "log_probs looks like (1287,)\n", + "logs prob looks like torch.Size([1287])\n", + "torch.from_numpy(rewards) looks like torch.Size([1287])\n", + "rewards looks like (1718,)\n", + "log_probs looks like (1718,)\n", + "logs prob looks like torch.Size([1718])\n", + "torch.from_numpy(rewards) looks like torch.Size([1718])\n", + "rewards looks like (1859,)\n", + "log_probs looks like (1859,)\n", + "logs prob looks like torch.Size([1859])\n", + "torch.from_numpy(rewards) looks like torch.Size([1859])\n", + "rewards looks like (1215,)\n", + "log_probs looks like (1215,)\n", + "logs prob looks like torch.Size([1215])\n", + "torch.from_numpy(rewards) looks like torch.Size([1215])\n", + "rewards looks like (1181,)\n", + "log_probs looks like (1181,)\n", + "logs prob looks like torch.Size([1181])\n", + "torch.from_numpy(rewards) looks like torch.Size([1181])\n", + "rewards looks like (1378,)\n", + "log_probs looks like (1378,)\n", + "logs prob looks like torch.Size([1378])\n", + "torch.from_numpy(rewards) looks like torch.Size([1378])\n", + "rewards looks like (1851,)\n", + "log_probs looks like (1851,)\n", + "logs prob looks like torch.Size([1851])\n", + "torch.from_numpy(rewards) looks like torch.Size([1851])\n", + "rewards looks like (2218,)\n", + "log_probs looks like (2218,)\n", + "logs prob looks like torch.Size([2218])\n", + "torch.from_numpy(rewards) looks like torch.Size([2218])\n", + "rewards looks like (2502,)\n", + "log_probs looks like (2502,)\n", + "logs prob looks like torch.Size([2502])\n", + "torch.from_numpy(rewards) looks like torch.Size([2502])\n", + "rewards looks like (1642,)\n", + "log_probs looks like (1642,)\n", + "logs prob looks like torch.Size([1642])\n", + "torch.from_numpy(rewards) looks like torch.Size([1642])\n", + "rewards looks like (1892,)\n", + "log_probs looks like (1892,)\n", + "logs prob looks like torch.Size([1892])\n", + "torch.from_numpy(rewards) looks like torch.Size([1892])\n", + "rewards looks like (2003,)\n", + "log_probs looks like (2003,)\n", + "logs prob looks like torch.Size([2003])\n", + "torch.from_numpy(rewards) looks like torch.Size([2003])\n", + "rewards looks like (3407,)\n", + "log_probs looks like (3407,)\n", + "logs prob looks like torch.Size([3407])\n", + "torch.from_numpy(rewards) looks like torch.Size([3407])\n", + "rewards looks like (3425,)\n", + "log_probs looks like (3425,)\n", + "logs prob looks like torch.Size([3425])\n", + "torch.from_numpy(rewards) looks like torch.Size([3425])\n", + "rewards looks like (1840,)\n", + "log_probs looks like (1840,)\n", + "logs prob looks like torch.Size([1840])\n", + "torch.from_numpy(rewards) looks like torch.Size([1840])\n", + "rewards looks like (1529,)\n", + "log_probs looks like (1529,)\n", + "logs prob looks like torch.Size([1529])\n", + "torch.from_numpy(rewards) looks like torch.Size([1529])\n", + "rewards looks like (1407,)\n", + "log_probs looks like (1407,)\n", + "logs prob looks like torch.Size([1407])\n", + "torch.from_numpy(rewards) looks like torch.Size([1407])\n", + "rewards looks like (2541,)\n", + "log_probs looks like (2541,)\n", + "logs prob looks like torch.Size([2541])\n", + "torch.from_numpy(rewards) looks like torch.Size([2541])\n", + "rewards looks like (1194,)\n", + "log_probs looks like (1194,)\n", + "logs prob looks like torch.Size([1194])\n", + "torch.from_numpy(rewards) looks like torch.Size([1194])\n", + "rewards looks like (1431,)\n", + "log_probs looks like (1431,)\n", + "logs prob looks like torch.Size([1431])\n", + "torch.from_numpy(rewards) looks like torch.Size([1431])\n", + "rewards looks like (3340,)\n", + "log_probs looks like (3340,)\n", + "logs prob looks like torch.Size([3340])\n", + "torch.from_numpy(rewards) looks like torch.Size([3340])\n", + "rewards looks like (897,)\n", + "log_probs looks like (897,)\n", + "logs prob looks like torch.Size([897])\n", + "torch.from_numpy(rewards) looks like torch.Size([897])\n", + "rewards looks like (1821,)\n", + "log_probs looks like (1821,)\n", + "logs prob looks like torch.Size([1821])\n", + "torch.from_numpy(rewards) looks like torch.Size([1821])\n", + "rewards looks like (1906,)\n", + "log_probs looks like (1906,)\n", + "logs prob looks like torch.Size([1906])\n", + "torch.from_numpy(rewards) looks like torch.Size([1906])\n", + "rewards looks like (2688,)\n", + "log_probs looks like (2688,)\n", + "logs prob looks like torch.Size([2688])\n", + "torch.from_numpy(rewards) looks like torch.Size([2688])\n", + "rewards looks like (1169,)\n", + "log_probs looks like (1169,)\n", + "logs prob looks like torch.Size([1169])\n", + "torch.from_numpy(rewards) looks like torch.Size([1169])\n", + "rewards looks like (1444,)\n", + "log_probs looks like (1444,)\n", + "logs prob looks like torch.Size([1444])\n", + "torch.from_numpy(rewards) looks like torch.Size([1444])\n", + "rewards looks like (1376,)\n", + "log_probs looks like (1376,)\n", + "logs prob looks like torch.Size([1376])\n", + "torch.from_numpy(rewards) looks like torch.Size([1376])\n", + "rewards looks like (1395,)\n", + "log_probs looks like (1395,)\n", + "logs prob looks like torch.Size([1395])\n", + "torch.from_numpy(rewards) looks like torch.Size([1395])\n", + "rewards looks like (899,)\n", + "log_probs looks like (899,)\n", + "logs prob looks like torch.Size([899])\n", + "torch.from_numpy(rewards) looks like torch.Size([899])\n", + "rewards looks like (2152,)\n", + "log_probs looks like (2152,)\n", + "logs prob looks like torch.Size([2152])\n", + "torch.from_numpy(rewards) looks like torch.Size([2152])\n", + "rewards looks like (2294,)\n", + "log_probs looks like (2294,)\n", + "logs prob looks like torch.Size([2294])\n", + "torch.from_numpy(rewards) looks like torch.Size([2294])\n", + "rewards looks like (881,)\n", + "log_probs looks like (881,)\n", + "logs prob looks like torch.Size([881])\n", + "torch.from_numpy(rewards) looks like torch.Size([881])\n", + "rewards looks like (1050,)\n", + "log_probs looks like (1050,)\n", + "logs prob looks like torch.Size([1050])\n", + "torch.from_numpy(rewards) looks like torch.Size([1050])\n", + "rewards looks like (1294,)\n", + "log_probs looks like (1294,)\n", + "logs prob looks like torch.Size([1294])\n", + "torch.from_numpy(rewards) looks like torch.Size([1294])\n", + "rewards looks like (1419,)\n", + "log_probs looks like (1419,)\n", + "logs prob looks like torch.Size([1419])\n", + "torch.from_numpy(rewards) looks like torch.Size([1419])\n", + "rewards looks like (1433,)\n", + "log_probs looks like (1433,)\n", + "logs prob looks like torch.Size([1433])\n", + "torch.from_numpy(rewards) looks like torch.Size([1433])\n", + "rewards looks like (2196,)\n", + "log_probs looks like (2196,)\n", + "logs prob looks like torch.Size([2196])\n", + "torch.from_numpy(rewards) looks like torch.Size([2196])\n", + "rewards looks like (1811,)\n", + "log_probs looks like (1811,)\n", + "logs prob looks like torch.Size([1811])\n", + "torch.from_numpy(rewards) looks like torch.Size([1811])\n", + "rewards looks like (1418,)\n", + "log_probs looks like (1418,)\n", + "logs prob looks like torch.Size([1418])\n", + "torch.from_numpy(rewards) looks like torch.Size([1418])\n", + "rewards looks like (1536,)\n", + "log_probs looks like (1536,)\n", + "logs prob looks like torch.Size([1536])\n", + "torch.from_numpy(rewards) looks like torch.Size([1536])\n", + "rewards looks like (1353,)\n", + "log_probs looks like (1353,)\n", + "logs prob looks like torch.Size([1353])\n", + "torch.from_numpy(rewards) looks like torch.Size([1353])\n", + "rewards looks like (1260,)\n", + "log_probs looks like (1260,)\n", + "logs prob looks like torch.Size([1260])\n", + "torch.from_numpy(rewards) looks like torch.Size([1260])\n", + "rewards looks like (1514,)\n", + "log_probs looks like (1514,)\n", + "logs prob looks like torch.Size([1514])\n", + "torch.from_numpy(rewards) looks like torch.Size([1514])\n", + "rewards looks like (2095,)\n", + "log_probs looks like (2095,)\n", + "logs prob looks like torch.Size([2095])\n", + "torch.from_numpy(rewards) looks like torch.Size([2095])\n", + "rewards looks like (1695,)\n", + "log_probs looks like (1695,)\n", + "logs prob looks like torch.Size([1695])\n", + "torch.from_numpy(rewards) looks like torch.Size([1695])\n", + "rewards looks like (2109,)\n", + "log_probs looks like (2109,)\n", + "logs prob looks like torch.Size([2109])\n", + "torch.from_numpy(rewards) looks like torch.Size([2109])\n", + "rewards looks like (967,)\n", + "log_probs looks like (967,)\n", + "logs prob looks like torch.Size([967])\n", + "torch.from_numpy(rewards) looks like torch.Size([967])\n", + "rewards looks like (1231,)\n", + "log_probs looks like (1231,)\n", + "logs prob looks like torch.Size([1231])\n", + "torch.from_numpy(rewards) looks like torch.Size([1231])\n", + "rewards looks like (1355,)\n", + "log_probs looks like (1355,)\n", + "logs prob looks like torch.Size([1355])\n", + "torch.from_numpy(rewards) looks like torch.Size([1355])\n", + "rewards looks like (1351,)\n", + "log_probs looks like (1351,)\n", + "logs prob looks like torch.Size([1351])\n", + "torch.from_numpy(rewards) looks like torch.Size([1351])\n", + "rewards looks like (1674,)\n", + "log_probs looks like (1674,)\n", + "logs prob looks like torch.Size([1674])\n", + "torch.from_numpy(rewards) looks like torch.Size([1674])\n", + "rewards looks like (2394,)\n", + "log_probs looks like (2394,)\n", + "logs prob looks like torch.Size([2394])\n", + "torch.from_numpy(rewards) looks like torch.Size([2394])\n", + "rewards looks like (2296,)\n", + "log_probs looks like (2296,)\n", + "logs prob looks like torch.Size([2296])\n", + "torch.from_numpy(rewards) looks like torch.Size([2296])\n", + "rewards looks like (897,)\n", + "log_probs looks like (897,)\n", + "logs prob looks like torch.Size([897])\n", + "torch.from_numpy(rewards) looks like torch.Size([897])\n", + "rewards looks like (2389,)\n", + "log_probs looks like (2389,)\n", + "logs prob looks like torch.Size([2389])\n", + "torch.from_numpy(rewards) looks like torch.Size([2389])\n", + "rewards looks like (1798,)\n", + "log_probs looks like (1798,)\n", + "logs prob looks like torch.Size([1798])\n", + "torch.from_numpy(rewards) looks like torch.Size([1798])\n", + "rewards looks like (1232,)\n", + "log_probs looks like (1232,)\n", + "logs prob looks like torch.Size([1232])\n", + "torch.from_numpy(rewards) looks like torch.Size([1232])\n", + "rewards looks like (1173,)\n", + "log_probs looks like (1173,)\n", + "logs prob looks like torch.Size([1173])\n", + "torch.from_numpy(rewards) looks like torch.Size([1173])\n", + "rewards looks like (1165,)\n", + "log_probs looks like (1165,)\n", + "logs prob looks like torch.Size([1165])\n", + "torch.from_numpy(rewards) looks like torch.Size([1165])\n", + "rewards looks like (1602,)\n", + "log_probs looks like (1602,)\n", + "logs prob looks like torch.Size([1602])\n", + "torch.from_numpy(rewards) looks like torch.Size([1602])\n", + "rewards looks like (1164,)\n", + "log_probs looks like (1164,)\n", + "logs prob looks like torch.Size([1164])\n", + "torch.from_numpy(rewards) looks like torch.Size([1164])\n", + "rewards looks like (2235,)\n", + "log_probs looks like (2235,)\n", + "logs prob looks like torch.Size([2235])\n", + "torch.from_numpy(rewards) looks like torch.Size([2235])\n", + "rewards looks like (1038,)\n", + "log_probs looks like (1038,)\n", + "logs prob looks like torch.Size([1038])\n", + "torch.from_numpy(rewards) looks like torch.Size([1038])\n", + "rewards looks like (1698,)\n", + "log_probs looks like (1698,)\n", + "logs prob looks like torch.Size([1698])\n", + "torch.from_numpy(rewards) looks like torch.Size([1698])\n", + "rewards looks like (1436,)\n", + "log_probs looks like (1436,)\n", + "logs prob looks like torch.Size([1436])\n", + "torch.from_numpy(rewards) looks like torch.Size([1436])\n", + "rewards looks like (1223,)\n", + "log_probs looks like (1223,)\n", + "logs prob looks like torch.Size([1223])\n", + "torch.from_numpy(rewards) looks like torch.Size([1223])\n", + "rewards looks like (2006,)\n", + "log_probs looks like (2006,)\n", + "logs prob looks like torch.Size([2006])\n", + "torch.from_numpy(rewards) looks like torch.Size([2006])\n", + "rewards looks like (1162,)\n", + "log_probs looks like (1162,)\n", + "logs prob looks like torch.Size([1162])\n", + "torch.from_numpy(rewards) looks like torch.Size([1162])\n", + "rewards looks like (2239,)\n", + "log_probs looks like (2239,)\n", + "logs prob looks like torch.Size([2239])\n", + "torch.from_numpy(rewards) looks like torch.Size([2239])\n", + "rewards looks like (1104,)\n", + "log_probs looks like (1104,)\n", + "logs prob looks like torch.Size([1104])\n", + "torch.from_numpy(rewards) looks like torch.Size([1104])\n", + "rewards looks like (1389,)\n", + "log_probs looks like (1389,)\n", + "logs prob looks like torch.Size([1389])\n", + "torch.from_numpy(rewards) looks like torch.Size([1389])\n", + "rewards looks like (1419,)\n", + "log_probs looks like (1419,)\n", + "logs prob looks like torch.Size([1419])\n", + "torch.from_numpy(rewards) looks like torch.Size([1419])\n", + "rewards looks like (1526,)\n", + "log_probs looks like (1526,)\n", + "logs prob looks like torch.Size([1526])\n", + "torch.from_numpy(rewards) looks like torch.Size([1526])\n", + "rewards looks like (1618,)\n", + "log_probs looks like (1618,)\n", + "logs prob looks like torch.Size([1618])\n", + "torch.from_numpy(rewards) looks like torch.Size([1618])\n", + "rewards looks like (2276,)\n", + "log_probs looks like (2276,)\n", + "logs prob looks like torch.Size([2276])\n", + "torch.from_numpy(rewards) looks like torch.Size([2276])\n", + "rewards looks like (2973,)\n", + "log_probs looks like (2973,)\n", + "logs prob looks like torch.Size([2973])\n", + "torch.from_numpy(rewards) looks like torch.Size([2973])\n", + "rewards looks like (1418,)\n", + "log_probs looks like (1418,)\n", + "logs prob looks like torch.Size([1418])\n", + "torch.from_numpy(rewards) looks like torch.Size([1418])\n", + "rewards looks like (1273,)\n", + "log_probs looks like (1273,)\n", + "logs prob looks like torch.Size([1273])\n", + "torch.from_numpy(rewards) looks like torch.Size([1273])\n", + "rewards looks like (2355,)\n", + "log_probs looks like (2355,)\n", + "logs prob looks like torch.Size([2355])\n", + "torch.from_numpy(rewards) looks like torch.Size([2355])\n", + "rewards looks like (1308,)\n", + "log_probs looks like (1308,)\n", + "logs prob looks like torch.Size([1308])\n", + "torch.from_numpy(rewards) looks like torch.Size([1308])\n", + "rewards looks like (1403,)\n", + "log_probs looks like (1403,)\n", + "logs prob looks like torch.Size([1403])\n", + "torch.from_numpy(rewards) looks like torch.Size([1403])\n", + "rewards looks like (1794,)\n", + "log_probs looks like (1794,)\n", + "logs prob looks like torch.Size([1794])\n", + "torch.from_numpy(rewards) looks like torch.Size([1794])\n", + "rewards looks like (1101,)\n", + "log_probs looks like (1101,)\n", + "logs prob looks like torch.Size([1101])\n", + "torch.from_numpy(rewards) looks like torch.Size([1101])\n", + "rewards looks like (1165,)\n", + "log_probs looks like (1165,)\n", + "logs prob looks like torch.Size([1165])\n", + "torch.from_numpy(rewards) looks like torch.Size([1165])\n", + "rewards looks like (1162,)\n", + "log_probs looks like (1162,)\n", + "logs prob looks like torch.Size([1162])\n", + "torch.from_numpy(rewards) looks like torch.Size([1162])\n", + "rewards looks like (1317,)\n", + "log_probs looks like (1317,)\n", + "logs prob looks like torch.Size([1317])\n", + "torch.from_numpy(rewards) looks like torch.Size([1317])\n", + "rewards looks like (993,)\n", + "log_probs looks like (993,)\n", + "logs prob looks like torch.Size([993])\n", + "torch.from_numpy(rewards) looks like torch.Size([993])\n", + "rewards looks like (2078,)\n", + "log_probs looks like (2078,)\n", + "logs prob looks like torch.Size([2078])\n", + "torch.from_numpy(rewards) looks like torch.Size([2078])\n", + "rewards looks like (1419,)\n", + "log_probs looks like (1419,)\n", + "logs prob looks like torch.Size([1419])\n", + "torch.from_numpy(rewards) looks like torch.Size([1419])\n", + "rewards looks like (1354,)\n", + "log_probs looks like (1354,)\n", + "logs prob looks like torch.Size([1354])\n", + "torch.from_numpy(rewards) looks like torch.Size([1354])\n", + "rewards looks like (1216,)\n", + "log_probs looks like (1216,)\n", + "logs prob looks like torch.Size([1216])\n", + "torch.from_numpy(rewards) looks like torch.Size([1216])\n", + "rewards looks like (1661,)\n", + "log_probs looks like (1661,)\n", + "logs prob looks like torch.Size([1661])\n", + "torch.from_numpy(rewards) looks like torch.Size([1661])\n", + "rewards looks like (2095,)\n", + "log_probs looks like (2095,)\n", + "logs prob looks like torch.Size([2095])\n", + "torch.from_numpy(rewards) looks like torch.Size([2095])\n", + "rewards looks like (2455,)\n", + "log_probs looks like (2455,)\n", + "logs prob looks like torch.Size([2455])\n", + "torch.from_numpy(rewards) looks like torch.Size([2455])\n", + "rewards looks like (2383,)\n", + "log_probs looks like (2383,)\n", + "logs prob looks like torch.Size([2383])\n", + "torch.from_numpy(rewards) looks like torch.Size([2383])\n", + "rewards looks like (2222,)\n", + "log_probs looks like (2222,)\n", + "logs prob looks like torch.Size([2222])\n", + "torch.from_numpy(rewards) looks like torch.Size([2222])\n", + "rewards looks like (2269,)\n", + "log_probs looks like (2269,)\n", + "logs prob looks like torch.Size([2269])\n", + "torch.from_numpy(rewards) looks like torch.Size([2269])\n", + "rewards looks like (2995,)\n", + "log_probs looks like (2995,)\n", + "logs prob looks like torch.Size([2995])\n", + "torch.from_numpy(rewards) looks like torch.Size([2995])\n", + "rewards looks like (1474,)\n", + "log_probs looks like (1474,)\n", + "logs prob looks like torch.Size([1474])\n", + "torch.from_numpy(rewards) looks like torch.Size([1474])\n", + "rewards looks like (2666,)\n", + "log_probs looks like (2666,)\n", + "logs prob looks like torch.Size([2666])\n", + "torch.from_numpy(rewards) looks like torch.Size([2666])\n", + "rewards looks like (1386,)\n", + "log_probs looks like (1386,)\n", + "logs prob looks like torch.Size([1386])\n", + "torch.from_numpy(rewards) looks like torch.Size([1386])\n", + "rewards looks like (2039,)\n", + "log_probs looks like (2039,)\n", + "logs prob looks like torch.Size([2039])\n", + "torch.from_numpy(rewards) looks like torch.Size([2039])\n", + "rewards looks like (2172,)\n", + "log_probs looks like (2172,)\n", + "logs prob looks like torch.Size([2172])\n", + "torch.from_numpy(rewards) looks like torch.Size([2172])\n", + "rewards looks like (2070,)\n", + "log_probs looks like (2070,)\n", + "logs prob looks like torch.Size([2070])\n", + "torch.from_numpy(rewards) looks like torch.Size([2070])\n", + "rewards looks like (2534,)\n", + "log_probs looks like (2534,)\n", + "logs prob looks like torch.Size([2534])\n", + "torch.from_numpy(rewards) looks like torch.Size([2534])\n", + "rewards looks like (1660,)\n", + "log_probs looks like (1660,)\n", + "logs prob looks like torch.Size([1660])\n", + "torch.from_numpy(rewards) looks like torch.Size([1660])\n", + "rewards looks like (1406,)\n", + "log_probs looks like (1406,)\n", + "logs prob looks like torch.Size([1406])\n", + "torch.from_numpy(rewards) looks like torch.Size([1406])\n", + "rewards looks like (1472,)\n", + "log_probs looks like (1472,)\n", + "logs prob looks like torch.Size([1472])\n", + "torch.from_numpy(rewards) looks like torch.Size([1472])\n", + "rewards looks like (2711,)\n", + "log_probs looks like (2711,)\n", + "logs prob looks like torch.Size([2711])\n", + "torch.from_numpy(rewards) looks like torch.Size([2711])\n", + "rewards looks like (1529,)\n", + "log_probs looks like (1529,)\n", + "logs prob looks like torch.Size([1529])\n", + "torch.from_numpy(rewards) looks like torch.Size([1529])\n", + "rewards looks like (1867,)\n", + "log_probs looks like (1867,)\n", + "logs prob looks like torch.Size([1867])\n", + "torch.from_numpy(rewards) looks like torch.Size([1867])\n", + "rewards looks like (1218,)\n", + "log_probs looks like (1218,)\n", + "logs prob looks like torch.Size([1218])\n", + "torch.from_numpy(rewards) looks like torch.Size([1218])\n", + "rewards looks like (1345,)\n", + "log_probs looks like (1345,)\n", + "logs prob looks like torch.Size([1345])\n", + "torch.from_numpy(rewards) looks like torch.Size([1345])\n", + "rewards looks like (1188,)\n", + "log_probs looks like (1188,)\n", + "logs prob looks like torch.Size([1188])\n", + "torch.from_numpy(rewards) looks like torch.Size([1188])\n", + "rewards looks like (1945,)\n", + "log_probs looks like (1945,)\n", + "logs prob looks like torch.Size([1945])\n", + "torch.from_numpy(rewards) looks like torch.Size([1945])\n", + "rewards looks like (987,)\n", + "log_probs looks like (987,)\n", + "logs prob looks like torch.Size([987])\n", + "torch.from_numpy(rewards) looks like torch.Size([987])\n", + "rewards looks like (2017,)\n", + "log_probs looks like (2017,)\n", + "logs prob looks like torch.Size([2017])\n", + "torch.from_numpy(rewards) looks like torch.Size([2017])\n", + "rewards looks like (2001,)\n", + "log_probs looks like (2001,)\n", + "logs prob looks like torch.Size([2001])\n", + "torch.from_numpy(rewards) looks like torch.Size([2001])\n", + "rewards looks like (1335,)\n", + "log_probs looks like (1335,)\n", + "logs prob looks like torch.Size([1335])\n", + "torch.from_numpy(rewards) looks like torch.Size([1335])\n", + "rewards looks like (1343,)\n", + "log_probs looks like (1343,)\n", + "logs prob looks like torch.Size([1343])\n", + "torch.from_numpy(rewards) looks like torch.Size([1343])\n", + "rewards looks like (2834,)\n", + "log_probs looks like (2834,)\n", + "logs prob looks like torch.Size([2834])\n", + "torch.from_numpy(rewards) looks like torch.Size([2834])\n", + "rewards looks like (1391,)\n", + "log_probs looks like (1391,)\n", + "logs prob looks like torch.Size([1391])\n", + "torch.from_numpy(rewards) looks like torch.Size([1391])\n", + "rewards looks like (1852,)\n", + "log_probs looks like (1852,)\n", + "logs prob looks like torch.Size([1852])\n", + "torch.from_numpy(rewards) looks like torch.Size([1852])\n", + "rewards looks like (1256,)\n", + "log_probs looks like (1256,)\n", + "logs prob looks like torch.Size([1256])\n", + "torch.from_numpy(rewards) looks like torch.Size([1256])\n", + "rewards looks like (1184,)\n", + "log_probs looks like (1184,)\n", + "logs prob looks like torch.Size([1184])\n", + "torch.from_numpy(rewards) looks like torch.Size([1184])\n", + "rewards looks like (1939,)\n", + "log_probs looks like (1939,)\n", + "logs prob looks like torch.Size([1939])\n", + "torch.from_numpy(rewards) looks like torch.Size([1939])\n", + "rewards looks like (1274,)\n", + "log_probs looks like (1274,)\n", + "logs prob looks like torch.Size([1274])\n", + "torch.from_numpy(rewards) looks like torch.Size([1274])\n", + "rewards looks like (1367,)\n", + "log_probs looks like (1367,)\n", + "logs prob looks like torch.Size([1367])\n", + "torch.from_numpy(rewards) looks like torch.Size([1367])\n", + "rewards looks like (1284,)\n", + "log_probs looks like (1284,)\n", + "logs prob looks like torch.Size([1284])\n", + "torch.from_numpy(rewards) looks like torch.Size([1284])\n", + "rewards looks like (1127,)\n", + "log_probs looks like (1127,)\n", + "logs prob looks like torch.Size([1127])\n", + "torch.from_numpy(rewards) looks like torch.Size([1127])\n", + "rewards looks like (1298,)\n", + "log_probs looks like (1298,)\n", + "logs prob looks like torch.Size([1298])\n", + "torch.from_numpy(rewards) looks like torch.Size([1298])\n", + "rewards looks like (1638,)\n", + "log_probs looks like (1638,)\n", + "logs prob looks like torch.Size([1638])\n", + "torch.from_numpy(rewards) looks like torch.Size([1638])\n", + "rewards looks like (1144,)\n", + "log_probs looks like (1144,)\n", + "logs prob looks like torch.Size([1144])\n", + "torch.from_numpy(rewards) looks like torch.Size([1144])\n", + "rewards looks like (1370,)\n", + "log_probs looks like (1370,)\n", + "logs prob looks like torch.Size([1370])\n", + "torch.from_numpy(rewards) looks like torch.Size([1370])\n", + "rewards looks like (1835,)\n", + "log_probs looks like (1835,)\n", + "logs prob looks like torch.Size([1835])\n", + "torch.from_numpy(rewards) looks like torch.Size([1835])\n", + "rewards looks like (2149,)\n", + "log_probs looks like (2149,)\n", + "logs prob looks like torch.Size([2149])\n", + "torch.from_numpy(rewards) looks like torch.Size([2149])\n", + "rewards looks like (1033,)\n", + "log_probs looks like (1033,)\n", + "logs prob looks like torch.Size([1033])\n", + "torch.from_numpy(rewards) looks like torch.Size([1033])\n", + "rewards looks like (989,)\n", + "log_probs looks like (989,)\n", + "logs prob looks like torch.Size([989])\n", + "torch.from_numpy(rewards) looks like torch.Size([989])\n", + "rewards looks like (1900,)\n", + "log_probs looks like (1900,)\n", + "logs prob looks like torch.Size([1900])\n", + "torch.from_numpy(rewards) looks like torch.Size([1900])\n", + "rewards looks like (1706,)\n", + "log_probs looks like (1706,)\n", + "logs prob looks like torch.Size([1706])\n", + "torch.from_numpy(rewards) looks like torch.Size([1706])\n", + "rewards looks like (1235,)\n", + "log_probs looks like (1235,)\n", + "logs prob looks like torch.Size([1235])\n", + "torch.from_numpy(rewards) looks like torch.Size([1235])\n", + "rewards looks like (2693,)\n", + "log_probs looks like (2693,)\n", + "logs prob looks like torch.Size([2693])\n", + "torch.from_numpy(rewards) looks like torch.Size([2693])\n", + "rewards looks like (1021,)\n", + "log_probs looks like (1021,)\n", + "logs prob looks like torch.Size([1021])\n", + "torch.from_numpy(rewards) looks like torch.Size([1021])\n", + "rewards looks like (1126,)\n", + "log_probs looks like (1126,)\n", + "logs prob looks like torch.Size([1126])\n", + "torch.from_numpy(rewards) looks like torch.Size([1126])\n", + "rewards looks like (1334,)\n", + "log_probs looks like (1334,)\n", + "logs prob looks like torch.Size([1334])\n", + "torch.from_numpy(rewards) looks like torch.Size([1334])\n", + "rewards looks like (1337,)\n", + "log_probs looks like (1337,)\n", + "logs prob looks like torch.Size([1337])\n", + "torch.from_numpy(rewards) looks like torch.Size([1337])\n", + "rewards looks like (1502,)\n", + "log_probs looks like (1502,)\n", + "logs prob looks like torch.Size([1502])\n", + "torch.from_numpy(rewards) looks like torch.Size([1502])\n", + "rewards looks like (2059,)\n", + "log_probs looks like (2059,)\n", + "logs prob looks like torch.Size([2059])\n", + "torch.from_numpy(rewards) looks like torch.Size([2059])\n", + "rewards looks like (2057,)\n", + "log_probs looks like (2057,)\n", + "logs prob looks like torch.Size([2057])\n", + "torch.from_numpy(rewards) looks like torch.Size([2057])\n", + "rewards looks like (1300,)\n", + "log_probs looks like (1300,)\n", + "logs prob looks like torch.Size([1300])\n", + "torch.from_numpy(rewards) looks like torch.Size([1300])\n", + "rewards looks like (3078,)\n", + "log_probs looks like (3078,)\n", + "logs prob looks like torch.Size([3078])\n", + "torch.from_numpy(rewards) looks like torch.Size([3078])\n", + "rewards looks like (1724,)\n", + "log_probs looks like (1724,)\n", + "logs prob looks like torch.Size([1724])\n", + "torch.from_numpy(rewards) looks like torch.Size([1724])\n", + "rewards looks like (1468,)\n", + "log_probs looks like (1468,)\n", + "logs prob looks like torch.Size([1468])\n", + "torch.from_numpy(rewards) looks like torch.Size([1468])\n", + "rewards looks like (2674,)\n", + "log_probs looks like (2674,)\n", + "logs prob looks like torch.Size([2674])\n", + "torch.from_numpy(rewards) looks like torch.Size([2674])\n", + "rewards looks like (1376,)\n", + "log_probs looks like (1376,)\n", + "logs prob looks like torch.Size([1376])\n", + "torch.from_numpy(rewards) looks like torch.Size([1376])\n", + "rewards looks like (1564,)\n", + "log_probs looks like (1564,)\n", + "logs prob looks like torch.Size([1564])\n", + "torch.from_numpy(rewards) looks like torch.Size([1564])\n", + "rewards looks like (1452,)\n", + "log_probs looks like (1452,)\n", + "logs prob looks like torch.Size([1452])\n", + "torch.from_numpy(rewards) looks like torch.Size([1452])\n", + "rewards looks like (1205,)\n", + "log_probs looks like (1205,)\n", + "logs prob looks like torch.Size([1205])\n", + "torch.from_numpy(rewards) looks like torch.Size([1205])\n", + "rewards looks like (1520,)\n", + "log_probs looks like (1520,)\n", + "logs prob looks like torch.Size([1520])\n", + "torch.from_numpy(rewards) looks like torch.Size([1520])\n", + "rewards looks like (1099,)\n", + "log_probs looks like (1099,)\n", + "logs prob looks like torch.Size([1099])\n", + "torch.from_numpy(rewards) looks like torch.Size([1099])\n", + "rewards looks like (1506,)\n", + "log_probs looks like (1506,)\n", + "logs prob looks like torch.Size([1506])\n", + "torch.from_numpy(rewards) looks like torch.Size([1506])\n", + "rewards looks like (1175,)\n", + "log_probs looks like (1175,)\n", + "logs prob looks like torch.Size([1175])\n", + "torch.from_numpy(rewards) looks like torch.Size([1175])\n", + "rewards looks like (1251,)\n", + "log_probs looks like (1251,)\n", + "logs prob looks like torch.Size([1251])\n", + "torch.from_numpy(rewards) looks like torch.Size([1251])\n", + "rewards looks like (1318,)\n", + "log_probs looks like (1318,)\n", + "logs prob looks like torch.Size([1318])\n", + "torch.from_numpy(rewards) looks like torch.Size([1318])\n", + "rewards looks like (1446,)\n", + "log_probs looks like (1446,)\n", + "logs prob looks like torch.Size([1446])\n", + "torch.from_numpy(rewards) looks like torch.Size([1446])\n", + "rewards looks like (1220,)\n", + "log_probs looks like (1220,)\n", + "logs prob looks like torch.Size([1220])\n", + "torch.from_numpy(rewards) looks like torch.Size([1220])\n", + "rewards looks like (1343,)\n", + "log_probs looks like (1343,)\n", + "logs prob looks like torch.Size([1343])\n", + "torch.from_numpy(rewards) looks like torch.Size([1343])\n", + "rewards looks like (1186,)\n", + "log_probs looks like (1186,)\n", + "logs prob looks like torch.Size([1186])\n", + "torch.from_numpy(rewards) looks like torch.Size([1186])\n", + "rewards looks like (1443,)\n", + "log_probs looks like (1443,)\n", + "logs prob looks like torch.Size([1443])\n", + "torch.from_numpy(rewards) looks like torch.Size([1443])\n", + "rewards looks like (1212,)\n", + "log_probs looks like (1212,)\n", + "logs prob looks like torch.Size([1212])\n", + "torch.from_numpy(rewards) looks like torch.Size([1212])\n", + "rewards looks like (1346,)\n", + "log_probs looks like (1346,)\n", + "logs prob looks like torch.Size([1346])\n", + "torch.from_numpy(rewards) looks like torch.Size([1346])\n", + "rewards looks like (2124,)\n", + "log_probs looks like (2124,)\n", + "logs prob looks like torch.Size([2124])\n", + "torch.from_numpy(rewards) looks like torch.Size([2124])\n", + "rewards looks like (1461,)\n", + "log_probs looks like (1461,)\n", + "logs prob looks like torch.Size([1461])\n", + "torch.from_numpy(rewards) looks like torch.Size([1461])\n", + "rewards looks like (1425,)\n", + "log_probs looks like (1425,)\n", + "logs prob looks like torch.Size([1425])\n", + "torch.from_numpy(rewards) looks like torch.Size([1425])\n", + "rewards looks like (1457,)\n", + "log_probs looks like (1457,)\n", + "logs prob looks like torch.Size([1457])\n", + "torch.from_numpy(rewards) looks like torch.Size([1457])\n", + "rewards looks like (1223,)\n", + "log_probs looks like (1223,)\n", + "logs prob looks like torch.Size([1223])\n", + "torch.from_numpy(rewards) looks like torch.Size([1223])\n", + "rewards looks like (1310,)\n", + "log_probs looks like (1310,)\n", + "logs prob looks like torch.Size([1310])\n", + "torch.from_numpy(rewards) looks like torch.Size([1310])\n", + "rewards looks like (2446,)\n", + "log_probs looks like (2446,)\n", + "logs prob looks like torch.Size([2446])\n", + "torch.from_numpy(rewards) looks like torch.Size([2446])\n", + "\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vNb_tuFYhKVK" + }, + "source": [ + "### 訓練結果\n", + "\n", + "訓練過程中，我們持續記下了 `avg_total_reward`，這個數值代表的是：每次更新 policy network 前，我們讓 agent 玩數個回合（episodes），而這些回合的平均 total rewards 為何。\n", + "理論上，若是 agent 一直在進步，則所得到的 `avg_total_reward` 也會持續上升，直至 250 上下。\n", + "若將其畫出來則結果如下：" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 281 + }, + "id": "wZYOI8H10SHN", + "outputId": "80307382-3743-4f70-e08a-66c5e92451da" + }, + "source": [ + "end = time.time()\n", + "plt.plot(avg_total_rewards)\n", + "plt.title(\"Total Rewards\")\n", + "plt.show()" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEICAYAAAC3Y/QeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOx9d5gcxZn++3XPzCbtSqucAyAJRAYhkjFgkgAbsLE5MLYB+36cbbizjW0MxhljY5992DjgA4wDDphzgiNLHBkEiCSShFYJZa3S5t0JXb8/uqu7uro6TFjtjqbe59GjnZ7u6uru6be+er9QxBiDhoaGhkZtwRjqDmhoaGho7Hlo8tfQ0NCoQWjy19DQ0KhBaPLX0NDQqEFo8tfQ0NCoQWjy19DQ0KhBaPLXqHkQESOi/Ya6H6WCiE4iog1D3Q+N6oImf41hCyLqFv5ZRNQnfL445JiKEiERPU5E/c45txPR34loUqXa19AYKmjy1xi2YIyN4P8AvAvgA8K2P+7Brlzp9GE/ACMA/GgPntsHIkoN1bk19i5o8teoOhBRHRH9hIg2Of9+4mxrAvAggMnCDGEyES0goueIaDcRbSainxNRptjzMsZ2A/gngMOEvuxPRIuIaCcRrSCiC5zts5zzGc7n24hom3DcnUT0eefvy4jobSLqIqLVRPRvwn4nEdEGIvoKEW0B8BsiaiCi3xLRLiJ6C8BR0v35ChFtdNpbQUSnFHutGns/NPlrVCOuA3AMbBI+FMACAF9jjPUAOBPAJmGGsAlAAcAXAIwFcCyAUwB8ttiTEtEYAB8C0OZ8bgKwCMCfAIwHcCGAXxLRPMbYGgCdAA53Dn8vgG4iOsD5fCKAJ5y/twF4P4AWAJcBuImIjhBOPRHAaAAzAFwO4JsA9nX+nQHgEqGPcwFcCeAoxliz8/3aYq9VY++HJn+NasTFAL7DGNvGGGsH8G0AHw/bmTH2EmNsCWMszxhbC+C/YZNvUtxMRB0AtsMeQP7d2f5+AGsZY79x2n4FwN8AfMT5/gkAJxLRROfzX53Ps2AT/WtO/+5njK1iNp4A8AiAE4TzWwC+yRgbYIz1AbgAwA2MsZ2MsfUAbhb2LQCoAzCPiNKMsbWMsVVFXKtGjUCTv0Y1YjKAdcLndc42JYhoDhHdR0RbiKgTwPdgk3hS/AdjbCSAQwC0ApjqbJ8B4GhH3tlNRLthD0yc7J8AcBJsq/9JAI/DHnROBPAUY8xy+ncmES1xpKPdAM6S+tfOGOuXrn+9dP0AAMZYG4DPA/gWgG1EdBcRhd4bjdqFJn+NasQm2MTLMd3ZBgCqMrW3AFgOYDZjrAXAVwFQsSdljL0O4LsAfkFEBJuAn2CMjRL+jWCMfcY55AnYFvxJzt9PAzgeguRDRHWwZws/AjCBMTYKwANS/+Rr2gxgmvB5utTPPzHG3gP7HjEAPyj2WjX2fmjy16hG/BnA14hoHBGNBfANAH9wvtsKYAwRjRT2b4atv3cT0f4APoPS8TsAEwCcA+A+AHOI6ONElHb+HcV1fcbYSgB9AD4Ge5DodPp3Pjy9PwNbpmkHkCeiMwGcHtOHuwFcS0StRDQVngwFIppLRO9zBpV+5/xWGdersZdCk79GNeK7AJYCWAbgdQAvO9vAGFsOe3BY7UgxkwF8CcBHAXQBuA3AX0o9MWMsC+CnAL7OGOuCTdQXwp55bIFtZdcJhzwBYIejzfPP5PQZThv/AZvQdzn9vDemG9+GLfWsge0fuFP4rg7AjbD9E1tgO6KvLeFSNfZykF7MRUNDQ6P2oC1/DQ0NjRqEJn8NDQ2NGoQmfw0NDY0ahCZ/DQ0NjRpE2UWiiGgagN/DDn9jAG5ljP2UiEbDjqqYCTu9/ALG2C4nPvqnsBNZegFcyhh7OeocY8eOZTNnziy3qxoaGho1hZdeemk7Y2yc6rtKVAjMA/giY+xlImoG8BIRLQJwKYBHGWM3EtE1AK4B8BXYtVdmO/+Ohp2Ac3TUCWbOnImlS5dWoKsaGhoatQMiWhf2XdmyD2NsM7fcnZjltwFMAXAu7IQYOP+f5/x9LoDfO3VMlgAYpeuja2hoaOxZVFTzJ6KZsKsYPg87VX2z89UW2LIQYA8MYl2SDc42ua3LiWgpES1tb2+vZDc1NDQ0ah4VI38iGgG7RsnnnTR2F8zOJCsqm4wxditjbD5jbP64cUrJSkNDQ0OjRFSE/IkoDZv4/8gY+7uzeSuXc5z/+UIWG+EvSjXV2aahoaGhsYdQNvk70Tu/BvA2Y+y/hK/uhbfIxCUA7hG2f4JsHAOgQ5CHNDQ0NDT2ACoR7XM87IU0XieiV51tX4VdXOpuIvoU7CJUFzjfPQA7zLMNdqjnZRXog4aGhoZGESib/BljTyO8Nnpg7VBH/7+i3PNqaGhoaJQOneGroVFFyOYt3L10PSxLV+PVKA+VkH00NDT2EH7+WBtufnQlGtImPnCoXp1Ro3Roy19Do4qwrdNeyrerPz/EPdGodmjy19CoIljO4ktG0SsQa2j4oclfQ6OKwKV+gzT7a5QHTf4aGlUEbvlr7tcoF5r8NTSqCDrKR6NS0OSvoVFF4NyfLVhD2xGNqocmfw2NKgKXfbJ5Tf4a5UGTv4ZGFcHhfgxo8tcoE5r8NTSqCAVH9xnIafLXKA+a/DU0qggD+YLvfw2NUqHJX0OjitCX4+SvLX+N8qDJX0OjitDnyD3a8tcoF5r8NTSqCP1Zx/LXmr9GmdDkr6FRRejN2QXdtOyjUS40+WtoVBH6slr20agMNPlraFQR+rXDV6NC0OSvoVElYIx50T5a89coE5r8NTSqBNmC5SV5adlHo0xUhPyJ6A4i2kZEbwjbRhPRIiJa6fzf6mwnIrqZiNqIaBkRHVGJPmho7O3oF6x9LftolItKWf6/BbBQ2nYNgEcZY7MBPOp8BoAzAcx2/l0O4JYK9UFDY68GY145Z03+GuWiIuTPGHsSwE5p87kAfuf8/TsA5wnbf89sLAEwiogmVaIfGhqloDebR192+MsoAvdr2UejbAym5j+BMbbZ+XsLgAnO31MArBf22+Bs84GILieipUS0tL29fRC7qVHrmPeNh3H09xYPdTdiYYmWv3b4apSJPeLwZfZ8tagliBhjtzLG5jPG5o8bN26QeqahYaOzPz/UXYjEii1duP9125YyDdKyj0bZSA1i21uJaBJjbLMj62xztm8EME3Yb6qzTUNDIwRn/ORJ9+/6lKFlH42yMZiW/70ALnH+vgTAPcL2TzhRP8cA6BDkIQ0NjRikU4Yb8qmhUSoqYvkT0Z8BnARgLBFtAPBNADcCuJuIPgVgHYALnN0fAHAWgDYAvQAuq0QfNDRqBSmDoLlfo1xUhPwZYxeFfHWKYl8G4IpKnFdDo1xUowVtGlSV/dYYXtAZvho1jd7s8Hb0qpAy7NdWjPvX0CgWmvw1ahrVEN8vwzQIQHXOWjSGDzT5a9Q0eqqQ/FOmQ/7a8tcoA5r8NWoaPQPVKPvY5F8q9/dm81izvaeCPdKoRmjy16hp9Fah5W86mn+pss+nfrsUJ//ocffzrp4stnX1V6JrGlUETf4aNY2eqnT42pa/VaLp/9zqHQA8h/Hh1y/CghserUznNBKhoy+HzR19Q9oHTf4aNY3egWq0/B3yL7PCw3B0GC9duxN/efFdAEBnf26IezN4OP2mJ3Ds9/9vSPugyV+jplGNln/aLM/y5xiODuMP/+o5fOVvr2Pp2p044juLfNbxDfe/he/e95Zv/427+/Cjh1cMi7DX1e3dOPOnT2F3bzZ2362dA3ugR9HQ5K9Rs8gVLNy/rPoqi7ihnmUSXrkzh8HEpo5+5C2G7V0ekd721Brc/vQa336Pvr0VP3+sDZs7ht5n8YvHVuHtzZ1Y9NbWoe5KImjy16hZPPjGFjzxTvWVC+dJXuVa/vlhzP75gt23XEwfs0510+FQ5ZTsMbmoKKyhlN40+WvULCzhxXOM6apApTT/Ycz9yHHyjyH1rLNff27ofTf8N1TMoDyUsqMmf42ahVgW2WL+wSAJVmzpwpzrHsSGXb2V7lokyo324ShGNuoeyGPtHswNyBXsvuVjnkkub38/PMjfyb8o4hg54ODB1zfj8t8vrWCvwqHJX6NmwRdEv+z4mQCK19D//MK7yBYsPPzmntV4K1XeoRjZ52O3P4+ThNyAwYYr+xTiLH+bPIeT7GMxhkVvbcX27ninrmz5f+aPL+ORPeQz0OSvUbPoc6zF5jq7uO1wDH1UIW1WRvMvRvZ5df1uAHuumBy3/Pn/cfsNB8sfsNm/d6CA//f7pbjkjhdijwjLMN8T91mTv8Zeh1fe3YVdPfHhdryoW5ND/nESA8eO7gG8uamj9A6WCVfzL4IffvzIChzzPX8iVynRQv17aO1g7ujNx1n+ea75h++3paMfC3/yJJZv6Yxs66E3tqCrjNwCrvnzvq9uj5fJekLyTPaEIaLJX2Ovwwd/+SwuvHVJ7H79+QIyKcO1pAsxVibHWTc/hbNvftqd5qvwztYuXP3X1wblJU6VIPv87P/asKXTHw6Z9HpF9O0hC5uTes7iMwA1uQ+40T7h/Xp1/W4s39KF//jzK6H7rGrvxqf/8BKu/uuyyH5ZFsPtT61WWuxc88879zVqcOW5GrykeH+ugP2//qD7fVJDpBxo8tfYq8Cnyyu2dsXu258toCFtulUy48IKOeQEHdUU/bN/fBl3L92A1e3didosBqZb2G3PJ3ntqfUPOKnn8tHRPHxQGIiw/OtSNs29s7UblsWwur0bP3houc/Bz2eB63ZEO++Xb+nCd+9/G0+tDIYIc2OA9ykqgKAuZQLwqsq2dw34Zi+a/DU0ikQxL01/zrLJv8RCaYRw099zWPrb/PvLG7AzgSQVhUqVdC7mejmx7SltnZ+HO6XDZB1X9omw/MVZQ95i+MJfXsUtj6/CKmFgpoRhmu5go3Awc8ufh59GPR8+IIVp/nFyVyWgyV9jr0IxhNaXK6A+bbgySqnW1t9f3oi/vrQBAPDyu7vwbNt2V4/v6PM05PU7e3HV3a/hyj+9nLhty2IBC9JN8tqDtX3Szjn3VBVUTvZxDt0klr/4XPOWBXJIWpTBogZyf1vcFxG8d3wA4QNS1Dgik7/8LLTlr6FRJIp5aWzyN73QySI1cOZEdL+1uRNf+p/XAAAf+uWz+Ojtz7sW5HOrd2CrQzLcItySoBTBlo5+3LToHezz1Qfw8Tue931nFhnnL1qRolQkE87Ma+7Hs23blW3w2UYxK5+9sbEDd7+4PvH+IgYcss/FJHFlY2QhsQ3A/n20NqYB2BKPZTFs6+qPzc5ljKFgMSH/IDjY8AEkK8wKZl5zv1Keq0vbsg8fTOWZhGpwqTSGjPyJaCERrSCiNiK6Zqj6obF3oRgC73fInxNbseUOojRd/r7f/OhKnH7TkwA8WSCJXPPwm1vw00dXAgCeadvh+67YJC9RZhIJX3X8n154V9kGd4r3FiH7vP9nT+Pqv3kOVMYYTvuvJ/DPVzbGHstlnLxr+Qefzb5ffQCPLt/m218FUXorFBgaMjbxrtvRg8Vvb8V7fvAYOp0ZGgtJ0bpp8Urs+9UHXJ+HKgSVR/vITvEoJzm3/GWH9Z4ovTEk5E9EJoBfADgTwDwAFxHRvKHoi8behaiX5tm27T6dtz9nO3zNEmUfeX/RKhaJlUs/JiUn7e6IFcZMs7hon21dnoNa7HNRso9zzv4yZB+LASu3dePzf3k1dl8u47jlGxTkLvY/UvaRLP+ufvvertvRi21dA8jmLXdb2C350/PrAADbu7OBNjkM53ckz0K6+4PPkv9OeZJXtoYs/wUA2hhjqxljWQB3ATh3iPqisRchitA+evvzOOXHTwAAvvw/r+HFtbvQkPEcvsW+cPK5xPLDqtBE16mYwKiLIv9iLX8x07Tg07+D/oTwc5av+Rcz2AQt/+jzRlr+kubf6ZDxmu09AQduWAQVj87hg4TKUODHytJYp4L8+Qw1VPbZWy1/AFMAiGLgBmebCyK6nIiWEtHS9vbqq7yoMTQIs97FqIrO/hweW2HLBaLDt9hoH3nqv2m3p+WLjl4O3n6SEE2VtchhulU9E3XTJSzATzIWY7GzncdWbMNr63d7mn8J0T78uovJSOYyDyfBOF/DQM7C+p29ysFMLA6XLzBX4lnV3u0+Jz64hPWwPm3f8w6nVr9K9uH3Ur5HqoGc78vvTVD22Xst/1gwxm5ljM1njM0fN27cUHdHYw+iP1fALx9vKykkMsx6Xy8UX3ts+TZ3+l6fNl0ZpVhrqyDtv8ln+Xv9SJuE51btcLX+JO91lOWfLnKwEmchIjEVLBZoQ27xst+8iHN/8Yyr+Rfj8OXgkoZ4riffacd9yzaFHsPJ2JN9/PdaHkBXb+/BCT98DDctfifQlvhcC5ZN/lNGNcBiwEvrdgEQBkWh2fauASxxlrysdxy0u53BQiX78N+efI9UA3lBIv9akn02ApgmfJ7qbNPQwOsbO/DDh+xyBMUmMoUR+PqdHjGv2OIlgNWlzIDl39GXw2f/+FJsVI5snYXtnyswXHTbEry4ZieAymn+SS1pkVj6hCStgsUSJ7Zxv0gplr9L/kJ/P3HHC7jyT69g4271OrZunH+I7CMPWjyiSrWQijgQ27JPDu+dMxYA8OJa+5lwy1u8p5/+w0u48NYlbmAAAOzqtck/J53/w7c8izuX2H4B+R6pSkYELX9Z9tl7yf9FALOJaBYRZQBcCODeIeqLxjADf+GzBavoao1h1vD6nZ7lv0kgnN5s3iU2ThJ3PrcWD7y+Bb97bm1R54oqMQAAr22w6wElIv8I2ScVUs//xbU78S///VzAivRZ/lm/FZw0OooPwqVo/gNO5U1RkpnQUgcAuCskusiVfdw4/mhZhP9mVPXxxevv6s8jV2CYProJY0dk3PO4mr9w3LYue0BZubUbDdzy71U7fJc6MwhAQf6KgVy2/OXf+b2vbkoUElwOhoT8GWN5AFcCeBjA2wDuZoy9ORR90Rh+sEJi0Tv6cgFikxFmMW3Y5RG+qM3v7Ml6tX2cY/m+jc4LHwZZ942z1t7daRf6Klf2CVvJ6+q/LsPza3bi3Z3+EgUisfhkH5bc8ufXVkqGr0r2Gdlgx9qL/ghxlscH0mxIqKd8r/li76pCaaKEwqXEkQ1pZEyP/lzNX2h2v3EjAADLt3R6mj+XfSIeojjAAtHRPmHkf8cza3DRbfH1qcrBkGn+jLEHGGNzGGP7MsZuGKp+aAw/iCTBpYJ8wcKh334EV90dHSYYZvnv6BlwrTdRm9/dmxNCPe0XcLkjC+2I8TnImr9sRX/1rP1x0JQW9/M7W+0w0ySWf1jaPxBe3oGTWdDy9/bzkX8hqPmHeTy507SU2j4q2YeTtCjTiYQqW/7ioGMpZix8RqIaNMUBjpN/S0MKmZRHf57l77U7pbUBgC0T1rmyD3f4hg+afdI96orS/J17Is9sAIRKYpXCsHX4atQuRJLgUsEzq2zHW9zi2GEWWd5ibqy6OJ2eO7HZp/nnChbe2mSX/t3aWZzmL5PxvuNG4Ph9x7qf2514+yThlSqpgENV2G3mNfe7xezksEe/7CNo/owldixyjbtPEU//0BtbXL1bhaxb6MzbxgcR8fy+uH0e6ulsE3MVomYsqpmheA5OxI0Z053xAV6egPgIeXdWbut2yzHs7uUO3/D7JifCdQ/Ea/5ZVd4AAV/56zJc8cfk5UCKQWpQWtXQKAOqEgQPvbEFALBg1ujIY8MKYlkWQ8p52fMWw9gRdfjtZUdhv/Ej3MSvvMWwur3HfRHjyF+2muXPRF6UiIgkftokmn/Bsu+VrMPLx2ZDZJ9/u/MlXHXanPjOwLuvslUL2I5RAPj4MTOUx+byDN0DeV+CHa9mKQ6g4iCVK/hJ8e3NXi1+VZRSFFTRTinD8JN/Pij78GvuzebRavllKnHGIgclyM9Xno1YFnP3cWUfxaBqEGH19m5X5qs0NPlrDDuI/M2tab44i2q6nc1bmPO1B/H9Dx2MWWOb3O2MMbeIV8FiLmkCtlV10JSRAOBL8uILfhw0pSVQuhmwF4rhCDgdA+RPbikBEXGyT75gRUbVmILmf+3fX8ddUv0cmWzCHL4A8F+LgqGRKnAyLmW5xGyhgEvueMENqwS8AcmXfauwpu9fthl1qVddKQ6wrztueUdV3wGv/ymDfLJPv2v5i5FBXhim/GxVbYZBTvLyF5oLt/xNIuzuzWFfx/dQaWjZR2PYwVd/xnknuKWlijbhGaw3LXrHd6z4glrMT/7iQixibZ+3NnciYxo4bt+x2NbV75No1m7vwQd/+ayyn0Bw1mEQuX4G3/XFkH9PTESNW4W0wALEDwRDC0VyitPsRc27oLDKS1mcZiBv+YhfhBgyGSbl/P3ljegeyGOfcfbA/ptn1kbmG0Q9F66tp0zD5/B1LX9FO9lCUB7ztxlN/vJMzP/7jrD8DUJHXw6jnEJ0lYYmf41hB5/swx2+zkuicuhxJ96I+pTPqhJDLwsWc+PjAX8JX1Hzb9vajX3GNWFCSz1yBeZz1skOONkalCNSCFCSfxx/RkX6AN5g9dqG3crvZQejaCUXY7mLx3HLtJT486gILdFxG+d/OGCi7Tz/z4dX4Gf/1xa6X6eUXa2UfUxSOnzFWRnvT66gsPxF53RMiK/svC9Is4vvP/A27nhmTeA4g+ykMh4ZVWlo8tcYdlA5fPmL2KsI5eOWf3NdyheBI5JO3mJuTXrAq8AIwFfYras/j9bGjEsM4nRctqjlaB9ZqjGIUK+QfeIS16L0fsAbrDp61evNRsk+xZC/SHiyRl0MosjfF+0TQ/7jndwAwHOeq7BbJn9FAbiUQW4AAODdl+3dWbyxscPXt2zeCjzrtzd14tlV231thkGWdArSjPS/n1ytPi5vIZu3MFJb/hq1Al+oJyd/XgVRYRXzUg1NdSkfgch1bFKi5U+i5e/F+fflCmjImMjwpR2FFzeg3RZky18mf7XlH+fwjbP8ueYfllQmW/4i+cYloqkcniJKsvwj9Hk5+zYK45o98o9uUy6V4H3mVrrs8OXPrmAxvP9nT7t/8/bk6169vQcfve15X5thCJRuSDjgcflPW/4aNQNLJfsIGZyy5bzDsfxH1KV8A4f40hUs5pIm4Nf8XcvfcbQ2pL0wQDk7VIRMCAELMIT84xy+sbKP098wK16eOYgEW0ySnIpgZQs4CZJa/lGEDvhJMMrhq6qTw58DJ/l0iOwjwo04yluRJB2X+Ca3HbemgoxRDZnYfUqBJn+NYQeRX1zZx60MGdTWuexjGuSf4ouWvwXfNN/n8BVkn76sXcdFTf6y7ON/cVWyT106+IrFav4xso8ZR/7S4CGSahz552Kib0opOBZ2zsaM6ba3paMfC3/yVGQ7osM+ekDx9zFbsNwMXf7bMQ3yO3ylZ8dX7uLHR8ldcVJa0PL3a/6iBKmCtvw1agayQ8z+33uB5PotXPbpzxVCNf8CYy5pApLDV1gcpT9XQEPGkwSy+WCCEIdsfarIP2m9fBFR2b1if8MknE5pkBLvQ6zlH+OALUXzD7PSRzak3ee7YVevch8O0yB3JTTAu44ZYxpx9cK5kedTW/5ynL//mIG85baTzVuRZTDKsvwthqZMdMS9jvbRqBlYKs1fICKZHLnl35+zJM3fH+0jx/lzuHH+XPNPm8ikgpq/bPnLRCqHH8ZZdGGIyu4V+xtG5CqHL7/2OGklq4jwEVFqqKcKIxvSrh6vSoYTYRL5Bm/etx+efwimtTb69s0pNHbePi8NHYj2kQh8e/eA++xzcZZ/hMP36Fmjsb17AGfc9CTW7ehx+uOXNVVBASK05a9RMxDfM0sI9eTvvly8i6fc9+cLoZq/Hecvav4ekYiaPy/fm0Tzl8lRtsSJgCNntOKTx8/CpcfNDL9gCWXLPs7x97y6Ed+6901k85Zr+cZFpviXPBxch29LfVqQ86LbJYKf/AUSlwcOudxytsDcfTjJmwZFWv7v+cFjePnd3U7fou+byuGbMghrbzwbh04bBQBYsbULv356DSyL4VePrwIAZFIG8gUWOxvT0T4aNQNR9hFL33ILSJZ9vIqT/qgMbgHbcpAc7eMdz63i3mwBFoOP/LNR5B9j+RMRUqaBb3xgHqaMaoi7bKHf6hBOub9hhMQT4T5316v47bNrkStYrnUZZ/nnImSfjGlUNNSzpSHli6WPgiz78P1ThuHW3XG/Czh8LTfT2pV9DAPplNdenHTTm1MPyIwx5XPgA5XoVyAAT7Vtx1+W2ol5dSn7fsbJfHGyUKnQ5K8x7CDKPtwizBUsj/wVtVIA26oTyWlnTxZt27qw/9cfwpubOv0ZvsLxhkEg8gYLf7SP1568NKNMavJqU4ZidpEE3YpcBhG8rbAQQ5nIbMs3Wiri4KT60rpdAR9GfdqIDMdkivwMAPjJ4pWBfU2DUJc2XS1d9K2o0FSXUlr+phG0/OU+5oXrHxBmDHURlr8MuSyGdy6mfA789yNKS0Tk80nVpQz05QrIWwz/8b79cMH8qYF2UgYV9dspBpr8NYYcdy5Zh/uXbXY/++v58/8Fy18iR/6y9+cKPgtyR0/WV1bADJF9ANsS5HKJHefvkL9ACjL5B6p6yrV9hL+LI/+8z1/wkSP9pOA6fGMsf45svuDKPknIf2tnP86/5Vl85g/+apL1aTMy2sdXDiImJDRj2msnR1W15PjsSfvizk8tUGr+adNwid37TsrGLVgBh68c5x8nZ6kK2gH2/dykKL3Mr0uelWRMb6CqS5nub2p0UwZnHzI50E4mNXgUrQu7aQw5vv7PNwAAZx9yNgB1kleuwNBcb5N/bzaPbN7CRbctwaSR9a7c0p/3HHMG2U478eUTQz1lLjYNch2tDWnTlQTEwSQu/l6GaPkbxZB/fw7N9WmXGI7fbyzOO3wKLr7dTipKxSR59eUKPss7W7Awut5OkBqIkVd4ljPgOdI56tOmcmF68dj+fB692TxG1EVTSyZlIGUYnuyjGJQuWjAd+45rwr+esA8AYMPO4BrJpkGoS5nu3wWLBZLTcpbl1uMPK+8Qh95cwW1fxG+eWYNfPLYqsD+fDcjnEKVH8bumuhROnDMOa+xo6qQAACAASURBVG88GzOvud/dLg8elYQmf41hB9Hy538XLE+3zRUYdvVmA8XC+p0pNABMaKnHju6sL1Ii5dNf/WScMsi1/MM0/2LJX+VXSIKegQJGNnjkz2Upua0oqUL8rmeggMkjnQEjRtvOOSUFVGhIm24dJQ45YenMnz6J9Tv78No3T488T13KQNokX0SNjGP2GY1zD5viflbNntKmV6ahMWM6yzQqZB9ngBCreoqWfxwYA5rqzECW9+K3t4XuD8iyj9+PIhJ72GA5mJa/ln00hh18JZ2FUM9Gh/xXt3fjmbbtgeP6cwX35RrfUo8dPQO+yJmwqp6AvSi6q/mLso/gkIyTTGT4sojlE0agayDvG7RMIt9gxUkwm7dDOJvrg8QhVu/s6s+5A2esw9diofuoNH9fUpjFsN6xzsPWVeDIpAyfJa06p5wdrZo9iZo/d4zKy2vmCgyZlAGDbFI2DQIRIZ2AWBuFMEw+8xQRZ5lnJINDvF+y5a88XpO/Ri1BWd7B8sj/9qfX4Kq7X1Mc5023x42ow47urM9aF2UYWfNPGRTi8LVf1riIDBVKd/jm0NLgkYFBUl6CW4LajmB6/qun4Noz9wfgSVuiszYnJDnFDWDtXQOhi6rXpU305yzc/tRqt503NwmLrPjq9ERr6LblbwiWf3B/eS0E1QCaNr1on8Y6PjMMDlAZk1y5jBsBdQksf5H8m+qC8fhbYhb8kclbHOTqkpB/EbOTYqHJX2PYgBOKL9pHyPBtSMerlD0DeaQMwrjmDLZ3Z33hmabhWeMyjaQMw923Pm24JMqJRLUOaxyiyP/Hj6zAr58OlvG1r6GAFsHKNAzyWb1iW2nDQGMmhRGO9c+tYDnsNCn5AwisETDeKajG2/zu/W/jlsdX4ZE3t+D8W7z1DUTCV627ICKTMpEyyD2G3+f7/v09gT5zqBa0Ei1/Lp2oau+nTMO9b5z8xVDPMIgDkEqaWbfDn5ksk71I8NlCIdTyD5d9ohPAykFZ5E9EHyGiN4nIIqL50nfXElEbEa0gojOE7QudbW1EdE0559fw46V1u/D759YOdTdKxu4+W09WxfnnLYaGTPjPlfNs90AepkEY01SHnT0DvlIHpuGJJzKRmAa58fW2w9cfGinnFiSBaKjKksXP/q8N19/3lvK4XN7yhS/aso8HsTQ17yffo15ybHKIcf5FKFAAgO998GAcNm0UjhaW0Hx942489OYW337ijG23s9D5lSfvp/R31KUMmCYF4vxbm7wiZnIIp9LyF+L8uewjS0g5Z4bEZ0zc95NE8xcHoBEK2QcAPnT4FPzg/IOdPvj7XCeQd3/O8pG/+J0cscQxnGWfNwB8CMCT4kYimgfgQgAHAlgI4JdEZBKRCeAXAM4EMA/ARc6+GhXA/yxdj5sSLss3HGBZzLeuK8/UlTN8C5a95mnGNEPlE/7ic8t/2ugGWAx49V1vwRODvEShgMPXJLfoV33a0/x7swV86943saa9J/JaVN3yl5BIzrg5y/JN97lG7bYrtMXb5V9zspItbzHDtxj/AwAcMLkF/7zieB8xr27vcRe65xAtf/4s50xsxkxhaU2OTMpA2vB8CHyQrRfILiD7qDR/0743mZThyjKy5W/LPoZn8TuDgJHgPjQICVYjFLIPAMyb3ILZE5oBAI1SQpZcOTQn5DOIzziM5IdttA9j7G0gqJ8COBfAXYyxAQBriKgNwALnuzbG2GrnuLucfdUmkEZR6M0WSsrAHCo8sbIdn/zti+5nvk6vv7aPF8efMu1KjH1WUFJoqjPRPZB3Lf/j9h0LwF8nxxSiZgIOX4FYGjKe5n/va5vQtq0bD76xGVHIpIzgSl4iYccQzc8eXYkT5ozDYdNGoWAxXzVQwyDfQMJ9ABbzrFf+NbcgQ2Ufp86PSNSZlBEpB3GyFO/R6u09gXsoav58Fpc2SElgdSkDKZNgMft5c2tdJPwkDl9O6PUpA3VpEwb5NX9uOKQMw83z4NeRZAhsTEfLPgAwdkSd2w/ZL+BfJ7jg1/yFZxw2CxlM8h+slqcAEIXDDc62sO0BENHlRLSUiJa2t7cPUjf3LvDyBNWC3b1Z3+Ih72ztwhPvtEtVPb1ibfLqSyK45d89UEDaNDBtdKNvMXeAyz7cUg46fDka0qZTTgB419F0UyrBWYDKMScSfpzl/+NF7+C8XzwDxhhyBeZ76W2y9w8k/DO/H3Mn2pbnSXPHAwjKPpxU7XUN/H1RrTkgIi05SjnkcjxiJBBfZcw01PH0Ym2dnOVZxKIUIvdLNWNxyT9tos6p1CkmmPFcCNGPw59lkgmQ3+GrJv+mupT7PEzpdyL+LuQkxCSW/5A6fIloMRG9ofh37qD1CgBj7FbG2HzG2Pxx48YN5qn2GvTl8lVl+cvT86/f8yYuueOFQEE2bqWmTCP0JeHFr3Z0D7jkduXJ+/n2MYkiHb4cXGtOm4ZrqcUnLQUJNGmGrz9WnrcnyD7kj/Mn8qxgTqCHT2/FkmtPwceOngEgKPv4fAhSX8L0Zo6UwvKPuw6+lKIYjSNCrNJZsBhyBQumVMogkezjbLv8vfvgnMMm2+QvSCt8BlSf9iRDz4AozuHbHEr+pkv+8kJD/sqhVqjDN4zkh1TzZ4ydyhg7SPHvnojDNgKYJnye6mwL265RAfRmCz6rebgjbKDaJqzPes3fXse9r20CYFt5YS/JvEn24t4bdvW51uD5R07Fmu+f5X4WZR+ZR1KuDgyBJPwJOgDQooipB+Au+ygiSYbvV//xuuscBYCtTuigaAEbUlEzg8i1gsXEtYkj61HvOMVly79VqAwpW/BxBMPvQ9zspUcYcLjmb2fSBgdGwyC3vVzBJn95VicPGjL5pwRfyL+esA9OmjseadN23HOLn9dbakibvt8BkMzyF2cfoZZ/JuUbyESI93ZbVz+2dHi/bfH84u/6yBmtyuMrjcFq+V4AFxJRHRHNAjAbwAsAXgQwm4hmEVEGtlP43kHqQ82hL1soafGQOGTzVuxiG6UgLBZ8s1ArZSBvueUfUmZ4Ys7YEXWY2mpXzjSltXr5i2lEyD4qwhfJiId6toTUVq9TSCdJMnz/9Py7uOVxrzzAa+ttB7XP8pcyfAFv8JIHHe5wlGvRiM5amUTjpAV+T2RJQ4a43gEf0FKGAZVSJ2bY5gsWsgUroHuHPaOwz4A9GN69dAPO/Km9Khi3/OvSRuAZJ9L8Bctf7N8CIfKpqc50B4mJI+t9x4v3du2OXtzxjBfeK858RePgb585Dl88bY59jcWGZhWBshy+RPRBAD8DMA7A/UT0KmPsDMbYm0R0N2xHbh7AFYyxgnPMlQAeBmACuIMx9mZZV6DhohjL/52tXRjTlMGYEXWx+37lb8vwj1c24q3vnBGIZigHYZb/5g514kxUSn7KIBwydaRj+RuB7wZgv0j8HQvKPvaWjI/8vb87+3PIpMJlJxWB+kI9I15iMVGIz3rEcxtShq9BXtx/Sjpvgxvn73fgtjZm3IxamTTFWcYPP3wIrv7rMt/3cnx8GMRcCE/2UVelNIyg7BM3CMn3UPVb4G2sdqKzeCE3MXGPz/JUA7YMMdonZRJOnDMOHzh0Mo6c0YqTf/Q4AHvAnTyqAT+98DC8d/Y49OYKSPNEshBJ7ZxDJ2N/x0+jAn+uimCaiqHcaJ9/APhHyHc3ALhBsf0BAA+Uc14NNXqzBTBm645xP5rTb3oS45vr8MJ1p8a2u/jtrQBga6llriVdsBjW7+zFzLFNoZb/po5glUQgWIlRhGEQDpw8Eg+8viVQ0thn+ZM6zM8lOMFMFc/V1Z/H6KZMqCWmGhR8Dt8QRzXgL6C2w4l4SpvkRvSYBvnyEkQHsEzI3MHam8u7xwPAqIa0R/7SNWRSBr5w6hwsmDUa7VIxN7ntKNz/uhcR5ck+hvK3aJLnvM9Z9oImcXH3ySx//zb+WxA1fz6DOWG/sfjyGXPx9uZO3LdMHc0lWv6mQfjdJ+2gRXEmzIMNeB2iVuH4OjM4wDSkTdx80eH4c0gmNSBGWIXuUjZ0hm+FsKq9G5+765XYRSkGE3yqH+f05dKQqK0n2T/RPDkGP3pkBU760eNYv7PXV9tcBGNqPTaqEmPKIOzjRPfIMwduJfuSpaT2OfGIBKSKigkjwDjyj7L8+RrEgO2wBnhooueHkEtTuIuFKM7bkDbRly34+jqqMe1p3maQ/D936mwcu++YyNDCqAEMgK8sNy9KlzJIOWCahlduIV+wkCuw2IxbuR0l+Rsy+XshpG6cv2AMXHHyfhjX7J/9is59n+wjjMDiLKUxJP4fUD8fl9gjfhPesx88y1+Tf4Vw1d2v4Z5XN+H1jR1Dcn7GGHodKydO+im2VAFvrxL+hGdX7QAAtHcPRNZ/SSv05ZRhKB2rgP2yzBoXTCYCPJ4XyzvIvKHS/OUqnoYRPg2Pk32irGaf5d/NtXLPyWuQOs6f7ydDRf7N9WnPSRwRjhhF/sXUJ+LVP1MmKcsyGORl3D7dth3/eGVjrOUvt6MqHudbU6BguY7v+pTpSinyICY/u298YB7OPmSSfVxIlFQ6RB6UoSJ/vi3qfqaEAWqwoMm/CGTzFm59cpUyIYaXyh3MpIwoDOQtN+5aNKjX7ejBYd95BGu3exmqu3qzKAa8vUpEEvGfMmP+pCAZqnh+ed1V+buZY0LIn7wXif+tKukM+F9WuXa9HZ6o7q/qJU9a0plLJICfND2ZgiBOVcgX5x88b33aCKwqZhrkWvxhA599bLgVK17DU1efjMe/dJJyv1GNdjnq5voUJrU0KAfMlGD5X/cP26Efp/nLZCmXVwb8pax39WQ9zT8j1vaRBj/p2Yl5FOL77JMEE77nUQQf9R1/MwaR+3U9/2Lwu2fX4nsPLIdB5C4wwcEHBPHH0t41gLRJGNVYplCeAGJct0jSbdu6sbs3h3cdnR0AdhZL/hW0/EUeiLT8UwYgxarbtdvDyT+KuAApzj+EAEWCk+UzI0TCsPtWuuwjYnuPI/uYhnsunnCmaltN/qa7brGIMMtfJCHx9/v0V07GFkFCE6N9po1uBGDfR9km+N1lC/Dc6h04bd4EjGxMK6/dMChogccQapLIF5H8t3dnXcu/LuXJPnGWv2nAjVAynNIRdvlsdSRYHH7/yQX4tztfcvsiLkQj/i+Cv2ta9hkm4KUCVLKJamGNo25YjMOvXzTo/QL89dvFl573VfRF7C6S/PlgUskcAubU7AmDKqM2iRU1e/wIHDp1pO87EvYRpRTV+aKm8GKUjQzVjE/cM04v53Atf8M7l2iJcgQTljxw8pfLJIuObxHiRzHyZ2prI+bP9EIaVbMX1f2aOLIenz5xX+w7boR9XpX/Rojz54jzVSWRnURn/46eAXdG3pAxXeNAPq9sxYv3mwGY7gx0cjXVpHjvnHG46/Jj3M/8Ol3yVxA8f3Sa/IcZVD9RTv6yDLmncq7EWi6Wj/xtSUEk/109nszw0Bv+yowinmnbjmfatrvXUInsYf5TzhasSMtfpe2nTUN57wHvBVp01Ym458r3+L7j749Y1VN+pzg5R03nTQUJu/0tw+HL0ZQxvSgZwdo3KNhf/lkO9QRs2UdVUjklzG5OPWC8sm9R2b7KlbSkbb/62JGY0OKPdVddu2lQoO/cx7Jg5mhcetzMROeXIRphOwTLvyFtYoyT6xDl8wCAKaM8qcpizCV/MXu3WC3+0Gmj8NMLDwPgvYveYBzcn59rsBZvBzT5F4Wox5B1MgrllY5EvLh2p1u8rNIIk324Liouai1q/p/+w0tYs11dsfLi2593140F/L6EUsFfqmzeCo32AdQkXKp+yp+cv7BbiCUd0U6U7BNX2yfJSyzmXMiaf7GWv7j4DC9LYAjt3X7JUbj+vIMCfYuKfVda/tJzmj1hRGAfleZvEAXuNZ+l3v3pY/Gtcw4MHiPsP765Ttkf0UDZ0tnvq9TK768c7cSvIWMa+Ptnj8P8maNd3w5jDNOc5MENu9QhyEnBZx7c6HFlPcX94VLrYGr+mvwjwBjDd+97C23b7LLD7jNSmPMD7kIk4W195FfP+ci0kugNsfx5PXtxgWzRwQgk/4FVUva59Dcv4ran1IuZAGESAwVqp3BEkasX4SM6fNXni5Z9wksCKB2+wqYkJZ3HjvB8QynD8ElUAR9FlOafMl0ivfS4mXjqKyf7+iD7N0RSLTbaR7aiVX4X1aWbRrA9MUNYeX7hJjx59cl4/VtnROwNrNzajb5cARlnIRd+f/ulWRFf0YuB4YjprU6fueUPXHyMXS/ppLnl1RhrFIrrAd4gpJpFWFr2GRq88u4uHPO9R7Fiaxduf3oNLrnjBQDBCBERruwTQk78+7c2dyq/Lxd9OUHzZ6LsE9T85WifpHJOJWWfOKiLeJUWhihq/mGyD29btGRv+pdD8flTZ/vOURT5C38nkQnGFmH5u7KP4p7Upw138ZlpoxvdgANDIn9VLHlktI9ilvHd8/wWurKIW8izlGUf1VKOYe3Up81A4TcR8ya1YMXWTvTnCm6WLb+/chQXf3aiDMmNhILFMGdCM9beeLZbs79UNIYsSxk1gxnMDN+aJf+X1u0KdXz+ZPFKbOnsdxcC6ZXqpMg/UTHbL4wgeXXIwXqWPtnHYrjtydU47DuPqB2+0o8/KalblQj1DLl+mTRUFk/KoFAfShLLX4z2CTp87c+ir+GDh0/F50+dI7RDoQZAbIZvAvJvbVRb/qpBx5V9FIlR9WnTXbhelIVcy5/k/71joy3/4HcLD5qEJ758UuTxKgI7cc64oiJmgOKs4CNntGLl1m70DOTdkhdjHMtf/v3z2ZP42xJlnzD8xymz8auPHZm4T/LAKs/ERFiuUzhx80WjZsn//Fuexdk3Px3YvnxLpxvexjlRFckj4j0/eMz9O4wgeSjoYBVq8ss+wA0PvI3dvTnXyhE1/14peSmpnFOO5b+tqx/rd/aGkqcsXyRJ3fd9F2n5e9NrbyUvddsqS9pwB4/QU8Rq/uLfH3dkBBkj6v11ZMTqkzLxuaGeSsvf9HRlI9gHfq2urmwEBwgVwr7zh4oGrXGZ5Fd97ywcu++YRHWlws4Th4OnjsRA3sKKrd0u6Y5pss8ny55RA3fUb/6q0+Zg4UETE/dJrotlKGZeHKcdOAEAcNbBkxK3Xyxqkvz5A924O+jAWfiTp7Bia5ezn03YnPz5MxK5Uv5xyDXqOTj5l6PhrdjShZnX3I9nV20PfNcX4vBtd0o4iNmQPdkCJgvVB8P6LKMc8l9ww6M44YePhX6fqG6LYYCFxPtE3Vcv2geC7KO2/KMczVHko7JiwzJ8rz/vINxy8RGB/cWyAmmhvINczx/wJAoVcYkWpq8kgZRZyslfNEiiZIaw64+Lf//CqXPwL/O9Su68nSmjGnz7zRjTGHpuIJlvinf/0KmjANhVUrnlzzX/zhDZR8Q4Z2AKq+JaCmTZJ8ry339iC9beeDYOnDwy8F2lUJPkL0og27rUFSQBL1ImivRk6Yi/UPJ0UR5ASgEn/YcV4Zmy7MN/UO3O9YnX3JctYO7EZtzwwYN8fY5D0v0YY3jynXb1lDnk+uWqmmFZoWFdiJoV8G98Dt8wzV9J4uT+H/b8lJcaQv72eYKvXrNg+YtSj+2r8B/PgxDmKKJrxHBNVfau6Vq16r6FIczyFycfqufW2pTBDz58iPLYmy86HPuMbcLiq96Le694j3IfuW1euluFx790Ev74r0dj9vgRGOkQN78fvKy1vMC76pl/+qR98cPzD8F5hykXGiwJsuwTZfnvCdRkhq9IhG9t6sT4ufXK/WQLQfWIdvaonaei86h7IO8udp3kRdvc0YcJzfUBJyEnGNULJtZvt5i9DGBvtuBWifTJPtk8GusaMdmxvKLi7UUktfz/Z+kGXP23ZfjxRw7F+UdOTXSMWFUzWwhJDIog+CQvUMowIlby8g8+/radPgrkX5cyfHKg6s5EhXqqZhg+y9/0wkoNg2CE3PtDp40KbBNJRrxn3PJ0SymzcKeiGHkUdg3uOYpIeJJxzqGTcc6hkxPvf+enFmBuhON1xpgmzHDKfBw1sxWL397mK9r3/06YhfftP8F3jMpPkTYNXHDUtMD2clCM5b8nUKPk771IUQtXd4aEnonSww6Z/J0XShxgLvvNC3hx7S4A8Zp/R18Ox37//3DJsTPw7XMPitxXhLhyU8HyyJ8PGGJ/erMFNAorG8llG97e3OkuhiEiqeXPHeDrVQvAhFnuIZEoIgyKsPwjCIiTm7+wm1pmUkamCI5X/uxl8uf3RlwcXWwpkFWqGMhGSJa/KPsUQn43k0YGreB6sR6NcF+4/GFKz13u25NfPhktDUFqCLvHe5K8TpidPNzyk8fPwkvrduHw6d4Aed3Z8wL7xRWUqxTk84gO/aFAjZK/JfztsYlcB14OCVMZX7Llz18ocR1RTvxAvOyzzVnY43fPrXPJf1tXP97cFB0iKss+tuPN678Y59+bLaAxY7qkJlv+f3tpg/IciatVO+129efR2Z9DS72nm8pTbg4e8+ytsiQ6Ku17H1UuO4nxKabtBzJ8I6bg4jF88KlPm77CYnx72jSQtwqBtpIsROK3/P3RPvLxFy2Y5vPziAiz/Bsky9/V/CXymR6ivcvJUe72ISKvOBy331i88o3TY/cbzKUSVeAzAHHp0KFATZK/aO1zQrnl8VX4wUPLffvJ5M+zd0UOki1/TqQDBfWLGfeiiINJX7aAhoyJi297Hiu3deO6sw4IPc5X3oGxwApCfss/j8a6VMAC5AgboOJkn5yzFB8//NdPr8Gvn16DtTee7e4TFjklxzyLs6vffXIBHlvejmmtjb7tk0bWu7X7oy1/5xwRcf485lz1eMT4eObcZlm/5f6NtEngP5viNX9hnV0p2kfWqb7/IbWGLvdNtOoDso8bSx7alA9hmn+SMNbhjLhqopXEXZcf4xbF4yhHNisHNenwFS3dbMHCW5s6A8QPBMmfzxJEIt3ZHWL5h0TQxGnTIvmvaredeut22PIJn5momgha/v5HmxX6niswNAoLk6za3uOrKR8W8REl+9y3bBNmX/cg3t3RG3mNYVmccj19caCZO7EZ3/jAPBgG4cKjpgMAHvnCe7H4qhOF40NP6Uk9BvmctyJShno74A0IYqatfH95d8XFysln+fvbVMk+8qpR4iI0xfCr6PAVE6lk2ef4/cYCAM48KFk4oW8wUmyvVuwp2QcAjtlnjBvlxKPsBrNmfxRqkvz9so+Fi29fotwvYPk7x4lhkzt7/KthuZq/YOGKzzbuQYszCW4l82lpnyRL5QoWlm+x5aDenGT5S/HW/Jr5INGQ8cj/6/98A8d871F337AeRln+P3xoBQDbWR01vslOdI6PObHvpzvxzeLYKYYrnnf4FKy98WzMmdCMpjpRI4+w/OGRaJjD192usvyFMgsc8szqhNmcSNVx3/KgorI2xQElLSx8rpJ9ohBm+fP1aHlbB0yywwnFxcijICeJcQzmIuN7AkO1Bocnuw3J6WuT/LNSnZtdvTlMaAkmnHQIySDZvFeFUoyc2dGTxaSR9ThgUgsAjyDFAUakzLAXZXV7Nz712xd9uQcDef8CMTL533D/21j4k6ewYVevL9qnYAV/0HmX/O39mgTZB1CntsuISgZ7d6fn3I0a31QLcADAwVNGYu2NZ2OWs+aAGCaaZOGMKAJSyT5hZKpKQvNkH2+bPLgePNXu/1EJiTStkJnEomqmSe5MIyrEVIU42Ue1AlYShC0tOFSWa6WwpzV/DrfGzxANnmVdNRH9JxEtJ6JlRPQPIholfHctEbUR0QoiOkPYvtDZ1kZE15Rz/lIhEjOPTDn1gAmB/USi6s3m3ePEl6e9awDTWhvdpB0+mouOTZEzw96T7z2wHI8u34Z7XtnobstKlj+vUMgJ6qmV7QDsUNLebMF1GBYslebPnOuwB5DGTPh6tGG/xbDFXMRch7hs6DBwAuH6pzjLSFIGIEltn6g4/6gqiqLswyGXPvZmB7FdBaDOKK6XLH+ximjJlr/o8HW2l/qM3Htc3VwfwJ6UfUTwlecuO37WkJy/3KteBOAgxtghAN4BcC0AENE8ABcCOBDAQgC/JCKTiEwAvwBwJoB5AC5y9t2jEPX49TttS5tbnGHoHsi7Gl2uYOHav7+OJ99pR3vXAMa11LkvhrePmijDrOpRjbazb5OwclJ71wC6+nOe7ONY7bwJPhjk8gx9AvnLsk9TxnQHI+4YjlqMvFjZR5yRZPNWZPhsGLyqmn5nJJBs4Yzo2j5i1IyzTdrHy6FQtC3E2/O2ZMvf9RkkZEYvmc3bVieRtrh+QHGWf0iop2P5l/J8xLbOODB5SYNqwFBZ/q1NGay98Wycd3jlEsmKQVnRPoyxR4SPSwB82Pn7XAB3McYGAKwhojYAC5zv2hhjqwGAiO5y9n2rnH4kQa5gr3G7tbMfdy9d727nseh81aEw9AwUkHOifVZv78GyDR14+M0tyOYtvHdEnWu5Woo4fxFhJNXa6EV6NNen0NWfx5f/ugxf++cb7mISsuzDP7uWf30K6Aw6fEePyLj94XXem+pSoVEaoZZ/TN0iwJ7xlGJZyssS+mZLCczpJE5HMVNWtqT56VQWtjt4iJq/RBieJBLbDQBCSKtI/r74fHKcvk4fijC3xaghUyH7hIXbxiGTMvDcte9za+TsLaj2aKVSUclQz08C+Ivz9xTYgwHHBmcbAKyXth+taoyILgdwOQBMnz697M6dftOTAGyHo+hU5Vr1TMHyv+q0OXht/W48unybu60vV3Ct+mUbOuxjxjTi5Xd3Y3xLXSBlPhcW0hjyQxsp1BCZNLIeXf12pM9A3lLIPnA+2+Tfm82jN5vHRKdeT4ExH3mObqrzHL7CsnbF6N7itckQyT6bD5J/ksxgVwaRio4lReKSziRtdGBFZLtyAjbIO0wm/7AoojDw6xTvtUhClsT7JwAAIABJREFURCTJPomaBQBMElbSSiuifUqVfQB1Ulm1YzDLJg9nxM53iGgxEb2h+HeusM91APIA/lipjjHGbmWMzWeMzR83rrxFFABgzfYerNneE4jLZ8wm3tFNXjr7afMm+D4Dts4vr9LFp9Hjm+tdgigoNH8RSWrDzJ3Y4vvOk338lj8n/55sAX25glsbxrKYS2b1aQONadNNOusTNP+w+OLQOP8kln/ech3Vqu/DIC8uXuzCMZHWG3f4Cpp/wPJPLPvY21TVK8OOV4GT8jH7eA7iwOpiVFr9F3GmJA6K5co+cfjgEMkXGqUh1vJnjJ0a9T0RXQrg/QBOYV6IxkYAYmGMqc42RGwfNETV5AaACS11vhjrupQRqMORdeLjRWzabevz9pJy9sscF+cfBnGwmD+jFf/72ib3M9eH+QLynAe4Qd3Rm0WuwHwOX07+LfVppExyJSJX9smkii4yF+bwFcl+IF/AQM5PLnLmtAqcpDJOffqwc4UhShpyHb5iVU9pH/4bUbUSFeq5z7gm3P6J+YF941CfNvHg507AjDGNmPeNh5X7GEShsfVxaKlPobM/L8k+9u9jMMhfTOTTqA6UG+2zEMDVAM5hjImFXO4FcCER1RHRLACzAbwA4EUAs4loFhFlYDuF7y2nD0kQFl7I0VSX8k2P69NmYC3TXIEFQuR4WOa45rpAqYQwzT9usRcAmD+z1fcdt/y7B7zQU1/EkZNoJjp8+Xma61PImIbbnz5B9gmz/MMKvYX1XST7AYXsI/sqVPCWFnQG0SKrR0dZ/qpFUWQy5faBsryDIr6dR9TUpUzsI/iLiuHoAya1BGq8y+c1ipSTOHgEiWjEcNmnVM1fY+9CuZr/zwHUAVjk/DiXMMY+zRh7k4juhu3IzQO4gjE7MZ6IrgTwMAATwB2MsTfL7EMknlrZjvHN6qqdHPXSFL4+bfrC7gBbw5dJkVtQ45rrXNnHiiF/7jc46T8fw0ULpuPfTtzX1xYAzJGqFnJ9ma/OxBiwS8hB2OFk5za5lr9Hns31abveTIFh5jX3u8c0Zkx0D6gHxTB/RegSlcK12g5fP9mHnUeEWNXTvoYiLf8Ecf7ifvL+3pqpqrad/4U8Af5M5EXoK5mpL0b7FIvPnzobFx89HeMF/Z/3WX4+ewqjGtOBd204QZ7t7+0oN9pnv4jvbgBwg2L7AwAeKOe8SdG2rRufuOOFUGssbRJyBRZYC7QuZSgsfyt00RPxR+Nq/iEEygeFtTt68f0Hl7vkz7ff9+/vCcQde5a/TaJ5i/nW4eV/8+soME/2OXjKSOzuy/msPSJ7wOvPhQxQIcQbKvvkoh2+XdLM6wfnH4xcgeFr/3zD3ebKPia3/Cun+XPKZiw8k9eKEP1VMfy8n/K9KiYqh+O1b56urHZqV/Ysujm7H0Q+4ge835FYdHBP4qWvnTYk502CJ798Mprqaov89+oM3/3Gj8B3zjnQterklYP4lFtO2KlLGYraOEGHL0dGqMDYmy3gij+9jDc2dij3LVhM6YPI5i1MbKnHQVOCK/dwouF+hILFfGUSdvXYf/NpPXf4zp/RiuvPOwhpk3zrEDekTV8kSaAvYZJV6OL0/jh/WfOXLf+0aQTKG3iWf2nkH6n5O18xFr6S1wGT7NnWvEnBWvF+2Yj7JoLJaOK5VPjUe2bh+vOCZbpHNqQxsjG4YlTUMyoFfFWq9zilKPY0xDLVww3TxzQWvaxktWOvr+p5wVHT8PV7bGVpv/EjfOUTGjMmOvpygaloygxa/vkCUzpxifwRFYvf2oq3NoeXX84VLCW55gosNNlEJqq8xXxrDbiWv9PngmVr/py0MqaBzj6PgN3qjiGZs2Hp/2GWvzjLGVBE+3RLlv/U1kZs3O2v9S9X9SxWlk4Sq83AvJBM6buFB03C4qvei/3GK8jfDfX07Ho+O5Nng1EyzdffX1w+oxjtUwmMqEvhqatPxnhFKRON2sNeT/5iSJ5cv4eToEz09nHBksgqyz9jGiDypuec+BvSptLRmbeYMs46m7dCyxgE1wm2XDLPmIa7IDWfwRQYg2V5pJU2DV9f3LruIUQVFqkk9mN3b9b2jaTN2Dh/0VENAEfOaA0sn5lyl1H0LP+ZYxqxpTN8mU0RSS3KqAJuKuIHoCyxkA7JR6hkyPi5h09RrtRVDuRywhq1i72e/AG7hvaLa3ZiW5e/AieXfRoU5C/Xaw/T/Lm1zotvcS5ozESQv0Jrt5O51Jqj7D8oCJb/pFH12OoQJO/zsg270ZPNuyUj5OUPm5zrDiPMcNnH+/ui257HsfuMwTc+MM/tX33aiNX8F8waDdOgQKSRN1B5pProF09S9kOFKPLng524Em4xjlRxJTCvv/7oLnd7Bdn/5LnjgbkVa05Dw4eaIP9j9hmDY/YZg2//rz+wiFvKsuYPBC3/bIEpI3jE/Uwi5B32DwtvzIfKPlao7CO3lbeYW3F0Qku9W++fk/8flrwLwCszLOvr8opOqj6qIMo+2zr78c7WLgBexmhzfdqO9smpo32+/6GD8WFnTV8ez88h1/O3hEXokyBKG//5R4/AXS+8iwMmNXuhk4lbVtfz5xnZnz1pX9++5VL/3f92bKCUuIbGYKAmyJ/jogXT8Ztn1rqfORFw0vzxRw5Fm7OAitLyF8ivKWOiJ1vwEavB1xtEOPlbLJipC9jWfSZE9pHb4pZ/fdrw1QSSZzByvRwOLneFcWuo7CNIHNm8hU0dtv+Ea/zN9Sm17ONY/iMb0r7FtEXIoZ7FxvlHaeNTRjXgi6fbJrQn+xQxsPAsW4PA6T1jGsrEpnJLBSStra+hUS726mgfGXMmNOP1b3lreoolEADg/COn4isL9weg0PzzftlnVKNd/iEtFePikINVvnjaHHz5DJuAxMgbt/0oy18aLHKO5t9Sn3YlHPs6ZPK3/w+Sv31MGFElSVAbyFvY0tEPxpjrr6hPmXaGb97CJ4+fhaVfs5PDueUv3h9Z9jGlgarYOP+k4MJPMRytKuwWhmEazKKhEUBNkT/gJ0huycdp/kR8+UOPFFubbItbtPyjyCGdMlwLUgx9XLJ6BwBbZw+rKx5m+bc0pH3O6oaM2pqW45fjklnCyJ/LPowxZAsWerMFdPblbX+FaSCTMrD47W3oHsijLm2495Br/uJAKZOkK/sYnsN3MOBa/kUINKriamG9q9UiYRrVh5ojf5FgOZnFRfukTQM5i/lkn1bH8hetdVl6EHkgbRqu5ds74JH5hbcuwdrtPY7sk8zy56GeLfUpXz/lYmNc9mmp98eQh2VZ/uCh5ZHJbFz2EWWdb9z7Bn799BrUpU3fvegdyLt94zWJxLr8wXslyz6DS/7FWOjuMUJ5iLDuactfo1pQc+QvgpO5LJcA8nqqhFzeb/mPUpC/7KAUZxQZk9yBp0eSfTZ39CNbhMPXTvLKo6Uh7eu7nKnskn+D37UTZpze8vgqPPD65ljLX3RYP/LmVgB2lI54z845bLJ7vU++Y684JkYdhXFkseUdzjtscqL9vPMWL/twiHH+oe1r8teoEtQ0+XOCkev4AP4BIZ0yXIvYi/awCVW01mVrVbS406Yn+/QM+Ml8d2+2SMvfciz/tC9SSR7E+Plkyz8K3QN5pcPXNEhZuoIPTGIfv3jaHBw5w3ZcniuQszjrCpNH+OzghNnJynjf9C+HYfX3zkq0L+CFlBYjz/DH6h/ci1upTUNjuEGTP4IWMxCUfbIFhrxl4dMn7ou1N57tSieitV6QSDMjtcFj2HukcgdbO/uLC/Us2OUdWhpSfstfdvhy8m/wk3+UotKfs5SWv0nkZt2qktQ6+/Nu9VQxg1QsZyAmsYVxpGEQHv/SSfjVx44M76QAvuhJUpRt+cfKPpr8NaoDNRXqKaPAwmWfep9kYycv5QrM1e15DRrRWpdr36QMQn3aQH/Osp2dDufLtW62dA44ETPJxuJcwUJHXw6jGjK+WUsw1NP+ny/ykgT9uYKysJthCIvThxSt4wu5i1VUm+u8c4vXF0WSM2PWUy4HpTh8OZKMMZr6NaoFNU3+liv7xFn+hL6c47R0Sd9f4AsIZnumTEJrYwabO/qRMQmMOQ7frMryD6/tI2NXbw4Wc0rkioOUvK5siMOXhcaq2OSvInfb8o8jfzs5aVyzZ/mLMoho+R88ZSQuPno6Zo1tchfF2RMgReROUiRJOtOWv0a1oKbJ33P4Bkk3ZfolG05QY53Kf3wQEAlBdlKahoGRDWls7uhH2jRc2aRH0vC3dPTbmr9A3j/88CH4yaJ3sKkjSIy8fv+oxowvdl4mJ050suUfJfv0ZQvKGkaG4ZE/T+qS6xfxzNSwwmGi5W8ahBs+eHB4RwYJ/A6V7vC1DwwP9SypWxoaexw1qfn/z6ePxV8uP8az/BWyj4i0abjVQHlxOD44iJatTP5px/IHbPJMRWj+cpz/BfOn4W+fPU7ZHz54tDamA8loInhz4kB2wuyx+Nyps0OP6c8XQh2+suwzY4y/SFiT4zsZ06Qm/1RCWWswYZQg+/CZkkHx5K7JX6NaUJOW/1Ez7UgUrtHLhc9kpFMG2p2icBOcBTK4hBF1pGmQmwzW1Z/HCCfZSiZ/3rZM5GGLhHOMakz7SjWrzi/jzk8dHdlmX9Z2+F5y7Axc+b7ZOOqGxXZbJFr+NvnPGtuE5Vu63GP/ecXxeHX97lB5JD0MguArJftoh69GtaMmyZ9jzoRmbO0cUGb4ihBr7ox39Gyvmmf4cSmD3BK6BcuCadgDgRzq6SZBSYOQOBiIFUM5RjVmlJE33jHFE1F/roBcwUJd2nSLlwH2zEW2/E/efzw6+nJYeNBEzBzThNkTmjF7QrAs8oi6FLoH8kUVahssuD0o4d4QEY7ZZwwefGMLZo5Vl0bW3K9RLahp8v/5R4/Amxs73IStMPA6NAbBXe2Hb4t611OGgS+cOgejGzP4wCGT8cKanQCCSV4ccpy/6ANorksFFqIf1ZCOrABZyipQ/bkC8gWGtEl+f4LC8j9o8khcMH9abJtfPmMuvnnvm+4aw0OJUjJ8OUyDcPHR03HqvAmBVeE4tOWvUS0Y+rdxCDGyIY3j9gtf0u6U/cdjn3FNWLHVrvQ5rrlOKD3M5YPwlz1lEurTprtOL9e8wxY0l+v5i+TdXJ8OkP/IhugFsUWCu/S4mYG1dFXoHsgjbzGkDMONnz9p7jis3NrtOsi5wzdpdNIlx83EJcfNTLTvnkIpoZ4m2cs4hhG/3a6GRnWgLA8cEV1PRMuI6FUieoSIJjvbiYhuJqI25/sjhGMuIaKVzr9Lyr2AwcSvLz0K1509z5V9JggLYhtewHgo5KUFeUG1XT1Z1e4BMhVrycgRO831KaRMQxmpJB7P8a1zDsSPLzg0vLMO+EyC9+XZa96HX33sSDvOXwr1jHI2D3cUY6BHrO2uaFfTv0Z1oNy39z8ZY4cwxg4DcB+AbzjbzwQw2/l3OYBbAICIRgP4JoCjASwA8E0iai2zD4MOHoUjkn9U5RlOnKZUtphr6Lt6PammScgunjyyHjL4ACKTP48iiopUKkX24Q7cMU12+5NHNaA+bdqyDwN29mTd2j7VTP6DF+dfQmc0NIYAZb29jDFxpfImeJx4LoDfMxtLAIwiokkAzgCwiDG2kzG2C8AiAAvL6cOeAJdrprZ6033GePhf8G3nFr7swJXLLACeDwEApo8JOhFNl/z9x452yDmKgEtd/LspY+KsQyYF2lq5tQtHXL8Iv392nXPuaEf5cEYpFnoS8teWv0a1oGzNn4huAPAJAB0ATnY2TwGwXthtg7MtbLuq3cthzxowffr0crtZFnodjX5qq0fOfJRTveoNaRO7kQuQRXNdKhC1M2ZEBu/utJdhnDQyqCXbjmUrUH+I685Rln8pzsf9xo/AhUdNC2QFm0RYvb0HALDCWb4xqeY/nFCMhCMjCbEPpuU/Y0wjZocsMq+hUSxiyZ+IFgOYqPjqOsbYPYyx6wBcR0TXArgStqxTNhhjtwK4FQDmz58/OMXdE2Jrl51l63P0RZAIDx2Va/UYBgWidkYLkUYqy5JvM4nwm0uPwv8u24S/v7wRk0fZElGk7FMkN2dMA4uvOjGkLQqUdahG8uco1eE7GO0mxRNfPjl+Jw2NhIh9exljpzLGDlL8u0fa9Y8Aznf+3ghAjAGc6mwL2z6ssa3TTsISZR8e86562aMWSB/prLnLHbVxBMo1f4PsuHq+bCOfJURJEUVb/hG7Rw1M1QRvDd/kx3DLI1Fht+q7JRo1inKjfcQ6AecCWO78fS+ATzhRP8cA6GCMbQbwMIDTiajVcfSe7mwb1tjmZOBOaw1q8qqXPUzzB4DmOpv8eQkEwyBccuwM/PDDhyjP7S4e7pxoa6c9C5mocA7LKJb8o8itGoleBS77lHI5SXwomvw1qgXlav43EtFcABaAdQA+7Wx/AMBZANoA9AK4DAAYYzuJ6HoALzr7fYcxtrPMPgw6rjx5P/z8sTbfilhzJtra64JZowP7NzjWuYowuRU5ZkQGG3f3IWUQvn3uQYH9OFzyd/7f5paZUNfPEVEs+UdJFt6SkMFks2rEYMk+OslLo1pQFvkzxs4P2c4AXBHy3R0A7ijnvHsaXzpjLr50xlzftiOmt2LJtacoLXBeYz9lBCdWPEHq48fMwJLVO/GVhXMD+4gwBdkHsIu5AUHn8BHTRwEAjpzRipfW7XKOjWw6gCSW/6HTRuGplduLa3gYoqSqngnupyZ/jWpBTWf4losw6YU7euUkLwAYyNlO02mjG/GRBKUReBucfH98wWF4ce1OTBacz8uvX+h+/7fPHIdLf/MCHl/RXnSoZ1Q0C7d6J7bEy03VgKJCMl2pSMs+GnsPNPkPAjjpqkoY87o445vjZRvAI31OVqObMjjjQH/wlRzxw/mneNknHLxu/4wxjbj9E/PdEtfVilI4OlmcfwkNa2gMATT5DwLcpR4VZDFvcguefKcdYxOSv1hUrlgUm+Ebtfuqdru+0dyJLTh13oTiOzPMUJLDd4hDPTU0KglN/oMAzhEqS/FnFx2O5Zs7A0lUYRDj/JOCk1QlZZ9eZwGZ/SfuHUlGpWTiJiH/vSQoSqMGUL1ZOsMQ/7ziePz0wsNc608V6jmyIY2j9xmTuE2+0EwxZFVM2eLj9vX6kmT/qIqW1YTSMnzj99EOX41qgSb/CuKwaaNw7mFT3GX/5MJupcCUHL5JwAeKJMfc+amj8bOLDvcdp8Lp8yZgamtDyfWChhuKGUz580xyhOZ+jWqBln0GESrLv1hwuacYzuW7JiE40yC3OFzU3rd+Yn7yDlQBSnkySe6nLuymUS3Qlv9gwAkNrERWrJzhmwSuzyHhMZ60VFzfqhmDJc/U0j3UqG5o8h8E8CxeVahnseDEXIzcwn0OSU/vEWHtMFcpi7kkgdb8NaoFmvwHAbzWvyrUs1iYJYR6esXLkh0kZxHXAgbrUmvpHmpUNzT5DyIqQf6pMkI9kx7D96slo3XQZJ8amj1pVDc0+Q8CPNmnfCIwXGIu3uObNNjIKMGvUPUYpFDPWrqFGtUNTf6DiEpYgXzyUJTD1z22ONmnFnjLq82vHb4atQ1N/oOAYhyEcfCyhYs5pjgGKml2UeUo5kqLeZw1NXvSqGpo8h9EVIIH+OyhqAxf5/+kg5BXPK6YnlUnvByIwWlfk79GtUCT/yCgkgsOc92+uAxf3o9kPdEO38qhhm6hRpVDk/8ggFVQ9+EWfzGBQ5zYknaDDzC1ZLUWc6kHTm4BALQ2ZirarobGUEKXdxgEcM6thIZerPNWPMYqVvZJ3q2qRSnP5rqzD8C5h03BfuNHxO5bS34TjeqGtvyHOdzyzEWVd+CWf3GyT01Z/kXsW5cyceSM1kHri4bGUECT/yCiElRaTHlm+Zik4lMtVXcYbIevhka1oCLkT0RfJCJGRGOdz0RENxNRGxEtI6IjhH0vIaKVzr9LKnH+YYcKenzdbN1SsoUT9oNPEGqBDwc7zl9Do1pQtuZPRNMAnA7gXWHzmQBmO/+OBnALgKOJaDSAbwKYD/s9fImI7mWM7Sq3H8MRlQn15G0VU97B/j9ptI9VxALlewtq50o1NNSohOV/E4Cr4bczzwXwe2ZjCYBRRDQJwBkAFjHGdjqEvwjAwgr0YVjh6H1GAwBmjmkqv7GSMnztfZM6fN3FSmqIEWvpWjU0VCjL8ieicwFsZIy9JlmmUwCsFz5vcLaFbVe1fTmAywFg+vTp5XRzj+Pjx8zAKQdMqMiSh57sk/wYV/MvUvapKcu/hq5VQ0OFWPInosUAJiq+ug7AV2FLPhUHY+xWALcCwPz58yuZNzXoIKKKrXVbzopTVkL2n9BSDwA473DlOLxXQlO/Rq0jlvwZY6eqthPRwQBmAeBW/1QALxPRAgAbAUwTdp/qbNsI4CRp++Ml9LtmUGzCFlC8pDG6KYPl1y90l3OsBQz2LGfa6L1joXuNvRclv+2MsdcZY+MZYzMZYzNhSzhHMMa2ALgXwCecqJ9jAHQwxjYDeBjA6UTUSkStsGcND5d/GXsvePZtUv0eAM45dDIA4Jh9xiQ+pj5t1pQUMpiX+uDnTsC9V7xn8E6goVEBDFaG7wMAzgLQBqAXwGUAwBjbSUTXA3jR2e87jLGdg9SHvQTFSTiATfprbzx7sDq0V6CSlVdlHDCpZfAa19CoECpG/o71z/9mAK4I2e8OAHdU6rx7O7ywTQ0NDY3KoXZE3iqFK08Mpqlag6ghhUtDQwlN/sMcPGZfU7+GhkYlocl/mIPLPlYxHl8NDQ2NGGjyH+bwYvaHuCMaGhp7FTT5D3MUW6FTQ0NDIwk0+Q9zuJq/dvhqaGhUEJr8hzlczV+Tf0WgB1ENDRua/Ic5ii3SpqGhoZEEmvyHOdwlGYe4H3sLaqmEhYZGFDT5D3Noy7+y0LKPhoYNTf7DHHPGNwMAZo5pHOKeaGho7E0YrMJuGhXCh46Ygn3Hj8Bh00YNdVc0NDT2ImjLf5iDiDTxDwK09K9R69Dkr1GT0NK/Rq1Dk7+GhoZGDUKTv4aGhkYNQpO/Rk1Bx/lraNjQ5K9RU9Bx/hoaNjT5a2hoaNQgNPlraGho1CDKIn8i+hYRbSSiV51/ZwnfXUtEbUS0gojOELYvdLa1EdE15ZxfQ6NUaOlfo9ZRiQzfmxhjPxI3ENE8ABcCOBDAZACLiWiO8/UvAJwGYAOAF4noXsbYWxXoh4ZGYmjpX6PWMVjlHc4FcBdjbADAGiJqA7DA+a6NMbYaAIjoLmdfTf4aGhoaexCV0PyvJKJlRHQHEbU626YAWC/s8//bO7sQK8owjv/+brqFRqUtIa3UKgshEZtsYfRB2JdZtAleCEFdBEUpFBGlCFEXXRT0CVFomfatWZEIUZZCV7mu5ceaaVZGiblWaHXTl08X82xO2zlHlz0zczzz/GA477wznPnt/8x5nHlnjvO991Xr/x+SbpPUJ6nvwIEDddAMgiPEsE9Qdo5a/CV9KKm/wtQDPAtMAbqAfcBj9RIzs8Vm1m1m3W1tbfV626DktJ7QAsCoqP5ByTnqsI+ZXXksbyRpCbDGZ/cCk1KL272PGv1BkDkPzz6XyW1jubQzDiiCcjPSu30mpmZnA/3eXg3MldQqqQPoBHqBjUCnpA5JY0guCq8eiUMQDIcJ41q5b+Y5tIyKI/+g3Iz0gu+jkrpInjK4B7gdwMy2S1pJciH3L2Cemf0NIGk+8D7QAiw1s+0jdAiCIAiGiY6Hn7t3d3dbX19f0RpBEATHFZI2mVl3pWXxC98gCIISEsU/CIKghETxD4IgKCFR/IMgCEpIFP8gCIISEsU/CIKghBwXt3pKOgB8O4K3OB34sU469SS8hkd4DY9G9YLGdWs2r7PMrOLP2Y+L4j9SJPVVu9e1SMJreITX8GhUL2hctzJ5xbBPEARBCYniHwRBUELKUvwXFy1QhfAaHuE1PBrVCxrXrTRepRjzD4IgCP5LWY78gyAIghRR/IMgCEpIUxd/STMl7ZS0W9KCgl32SNomabOkPu8bL2mtpC/99bSjvU+dXJZKGpDUn+qr6KKEpz3DrZKm5ez1oKS9nttmSbNSyxa6105J12ToNUnSekmfS9ou6S7vLzSzGl6FZibpREm9kra410Pe3yFpg29/hT/QCX/o0wrv3yDp7Jy9lkn6JpVXl/fntu/79lokfSZpjc9nm5eZNeVE8rCYr4DJwBhgCzC1QJ89wOlD+h4FFnh7AfBITi6XAdOA/qO5ALOA9wAB04ENOXs9CNxbYd2p/pm2Ah3+Wbdk5DURmObtk4Fdvv1CM6vhVWhm/neP8/ZoYIPnsBKY6/3PAXd4+07gOW/PBVZklFc1r2XAnArr57bv+/buAV4D1vh8pnk185H/hcBuM/vazP4A3gB6CnYaSg+w3NvLgRvz2KiZfQz8fIwuPcBLlvAJcKr++/jOrL2q0QO8YWa/m9k3wG6SzzwLr31m9qm3fwV2AGdScGY1vKqRS2b+d//ms6N9MmAGsMr7h+Y1mOMq4ApJdX/OZg2vauS270tqB64Dnvd5kXFezVz8zwS+S81/T+0vRtYY8IGkTZJu874zzGyft38AzihGraZLI+Q430+7l6aGxgrx8lPs80mOGhsmsyFeUHBmPoSxGRgA1pKcZRw0s78qbPtfL19+CJiQh5eZDeb1sOf1hKTWoV4VnOvNk8B9wGGfn0DGeTVz8W80LjGzacC1wDxJl6UXWnIO1xD33TaSC/AsMAXoAvYBjxUlImkc8BZwt5n9kl5WZGYVvArPzMz+NrMuoJ3k7OKcvB0qMdRL0rnAQhK/C4DxwP15Okm6Hhgws015breZi/9eYFJqvt37CsHM9vrrAPAOyRdi/+BppL8OFOVXw6XQHM1sv39hDwNLODJMkauXpNEkBfZVM3vbuwvPrJJXo2TmLgeB9cBFJMMmJ1TY9r9evvwU4KecvGb68JmZ2e87eR+UAAABeElEQVTAi+Sf18XADZL2kAxPzwCeIuO8mrn4bwQ6/Yr5GJILI6uLEJE0VtLJg23gaqDffW7x1W4B3i3Cz6nmshq42e98mA4cSg11ZM6QMdbZJLkNes31Ox86gE6gNyMHAS8AO8zs8dSiQjOr5lV0ZpLaJJ3q7ZOAq0iuR6wH5vhqQ/MazHEOsM7PpPLw+iL1D7hIxtXTeWX+OZrZQjNrN7OzSerUOjO7iazzqufV6kabSK7W7yIZb1xUoMdkkrsstgDbB11Ixuk+Ar4EPgTG5+TzOslwwJ8kY4m3VnMhudPhGc9wG9Cds9fLvt2tvtNPTK2/yL12Atdm6HUJyZDOVmCzT7OKzqyGV6GZAecBn/n2+4EHUt+DXpILzW8Crd5/os/v9uWTc/Za53n1A69w5I6g3Pb9lOPlHLnbJ9O84r93CIIgKCHNPOwTBEEQVCGKfxAEQQmJ4h8EQVBCovgHQRCUkCj+QRAEJSSKfxAEQQmJ4h8EQVBC/gFtzJkqOWV8fAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mV5jj4dThz0Y" + }, + "source": [ + "另外，`avg_final_reward` 代表的是多個回合的平均 final rewards，而 final reward 即是 agent 在單一回合中拿到的最後一個 reward。\n", + "如果同學們還記得環境給予登月小艇 reward 的方式，便會知道，不論**回合的最後**小艇是不幸墜毀、飛出畫面、或是靜止在地面上，都會受到額外地獎勵或處罰。\n", + "也因此，final reward 可被用來觀察 agent 的「著地」是否順利等資訊。" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 281 + }, + "id": "txDZ5vlGWz5w", + "outputId": "bc284774-255a-45ac-dabf-3dfb5e1e5565" + }, + "source": [ + "plt.plot(avg_final_rewards)\n", + "plt.title(\"Final Rewards\")\n", + "plt.show()\n" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEICAYAAAC3Y/QeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO29ebwlVXX3/Vt1zr23gQYapBm7mwZs1EYUsUViVIxgBKfWRH3xyavGPAb1wUQSfY1DHofHkNdX45NonIKPihonomJQiAgqkwpNM9NM3U1D0xPdTc/TPUOt94+qXbX3rr2r6gx1zzn3ru/ncz/nnBp31bln7VW/tfbaxMwQBEEQZhbBoBsgCIIgTD1i/AVBEGYgYvwFQRBmIGL8BUEQZiBi/AVBEGYgYvwFQRBmIGL8hZGEiPYQ0cl9OM4niOjf+9GmQUFENxDROwfdDmG0EOMvDDVE9BgR7Y+Nvfo7nplnM/OjFZ/7ZUQUxufcTUQPE9E7qjynIEwVYvyFUeC1sbFXfxum8NwbmHk2gMMA/A2ArxHRM6bw/AkUIb9ZoS/IP5IwkhARE9HT4/eXE9GXiOjq2EO/jYhO0bb9PBE9QUS7iOgOInpJp+fjiGsAbAPwnPi4ARF9iIhWE9FTRHQFER0Zr/sWEb0/fn9C3N6L48+nENG2eP8jiOjnRLSFiLbH7+dpbb+BiC4lot8C2AfgZCJ6BRE9REQ7ieiLAEjb/ulEdGO8bisR/bCL2yvMAMT4C9OFCwF8EsARAFYBuFRbdzuAMwAcCeB7AP6DiGZ1cvDYUL8OwFHx8QHgrwC8HsA5AI4HsB3Al+J1NwJ4Wfz+HACPAnip9vlmZg4R/Qa/CeBEAAsA7AfwRev0bwVwEYBDAewE8BMAfx+3ZTWAP9S2/RSAX8b3YR6Af+3kOoWZgxh/YRT4KRHtiP9+6tnmSmZexswtAN9FZOwBAMz878z8FDO3mPlzACYAlJVujieiHYiM8pUA/paZ74rXvRvAR5l5HTNPAvgEgDcSUR2R8X9xLNO8FMBnkBrpc+L1iNv1Y2bex8y7EXVa51htuJyZV8TXdgGAFcz8I2ZuAvgXAJu0bZuIOpLjmfkAM99S8jqFGYYYf2EUeD0zz4n/Xu/ZRjeA+wDMVh+I6ANE9GAshewAcDgir7kMG5h5DiLN/wsAXq6tOxHAlapjAvAggDaAY5h5NYC9iDqhlwD4OYANcbwgMf5EdDAR/RsRPU5EuwDcBGAOEdW08zyhvT9e/8xRZUZ9/QcRyUDLiGgFEf1FyesUZhhi/IVpTazvfxDAmwEcERvyndB08jLEnv3fATidiFQH9ASAC7SOaQ4zz2Lm9fH6GwG8EcB4vOxGAG9HJMncHW/zfkRPIS9k5sOQSkN6+/TSuxsBzNeuj/TPzLyJmf+SmY8H8C4AX1axEUHQEeMvTHcOBdACsAVAnYg+hsiL7xhmbgD4HICPxYu+CuBSIjoRAIhoLhEt1Xa5EcB7EXnzAHBD/PkWZm5r7dsPYEccLP54QTOuBnAaEf1JLC/9NYBj1UoiepMWMN6OqOMIO71WYfojxl+Y7lwL4BcAHgHwOIADMGWSTvkGgAVE9FoAnwdwFYBfEtFuALcCeKG27Y2IjLsy/rcAOFj7DESa/UEAtsb7/yLv5My8FcCbAHwawFMAFgH4rbbJCwDcRkR74ra9r+rxEMJoQjKZiyAIwsxDPH9BEIQZiBh/QRCEGYgYf0EQhBmIGH9BEIQZSH3QDSjDUUcdxQsXLhx0MwRBEEaKO+64Yyszz3WtGwnjv3DhQixfvnzQzRAEQRgpiOhx3zqRfQRBEGYgYvwFQRBmIGL8BUEQZiBi/AVBEGYgYvwFQRBmIJUafyKaT0S/IaIH4tri74uXH0lE1xHRyvj1iCrbIQiCIJhU7fm3ALyfmRcDOBvAxUS0GMCHAPyKmRcB+FX8WRAEQZgiKjX+zLyRme+M3+9GNNPRCQCWAvhWvNm3EM2DKghCAY1WiCuWP4EwlGq8Qm9MmeZPRAsBPA/AbYimudsYr9oE4BjH9hcR0XIiWr5ly5apaqYgDDVfvmEVPvije/GzezcMuinCiDMlxp+IZgP4MYBLmHmXvi6egzTjxjDzZcy8hJmXzJ3rHJ0sCDOObXsbAIDt8asgdEvlxp+IxhAZ/u8y80/ixU8S0XHx+uMAbK66HYIwHQgomtq3LaqP0CNVZ/sQgK8DeJCZ/7e26ipEE1kjfv3PKtshCNOFWhAZf9H8hV6purDbHwJ4K4D7iOjueNlHEM0/egUR/XdE86q+ueJ2CMK0QBn/tky/KvRIpcafmW8BQJ7V51Z5bkGYjiSyj3j+Qo/ICF9BGCFq8S9WZB+hV8T4C8IIkQZ8xfgLvSHGXxBGCGX8xfEXekWMvyCMEJLtI/QLMf6CMEIo498S4y/0iBh/QRghEs9fNH+hR8T4C8IIUZNUT6FPiPEXhBFEjL/QK2L8BWGEUCmeIvsIvSLGXxBGCOXxi+cv9IoYf0EYIVSKp3j+Qq+I8ReEEULJPuL5C70ixl8QRgjl+Uuev9ArYvwFYYRQnn9TZnMRekSMvyCMEO0wem22wsE2RBh5xPgLwggRJp6/GH+hN8T4C8IIoQK9TdH8hR4R4y8II0Ri/EX2EXpEjL8gjBAi+wj9Qoy/IIwQiecvxl/oETH+gjBCKM+/IameQo+I8ReEEUI8f6FfiPEXhBEiyfMX4y/0iBh/QRghkoCvZPsIPTIw409E5xPRw0S0iog+NKh2CMIooWQf0fyFXhmI8SeiGoAvAbgAwGIAbyGixYNoiyCMEm1J9RT6xKA8/7MArGLmR5m5AeAHAJYOqC2CMDKEEvAdKGHI+OZv12Bfo5W73VX3bMCTuw5MUau6Y1DG/wQAT2if18XLEojoIiJaTkTLt2zZMqWNE4RhRck+LZF9BsLKzXvwyZ89gJtXbvVuM9lq46+/fxeuvGv9FLasc4Y24MvMlzHzEmZeMnfu3EE3RxA64pu/XYOFH7oaO/c3+3pcFfBliPEfBK0weuIKc2orqUnWhn3CnUEZ//UA5muf58XLBGFa8J1bHwcAbNnd30d/ZVBkFsfBoO57nl1XHXReBzEMDMr43w5gERGdRETjAC4EcNWA2iIII4NSe4bbrExfOLn/xZ7/sH9H9UGclJlbRPReANcCqAH4BjOvGERbBGGUCBPPf9hNy/REGf28269WDftXNBDjDwDMfA2AawZ1fkEYRRLZZ8DtmKmU8ep5ROIyQxvwFYTpAfX1aCrPf9i9yulK6tXnyD7JNpU3pyfE+AtCFVT0w9eDiCL9TD1hic6X4yEYw/79iPEXhAqh/jr+iecPDL9nOR0pFfDFaEhzYvwFYYQwPP8BtmPmUsLzZxRuMwyI8ReEEcL0/IfcukxDyhj2RPMf8u5ZjL8gVEBVP3u9pM9wm5bpCVuvzm1GJCgvxl8QKqTPkr8h+4TDbl2mIWXGWag1Qz7AV4y/IFRBVZKMBHwHS5k0zlGpvyTGXxBGiGGvFzPdKZPtU0obGgLE+AvCCCGe/2DpqLxD9c3pCTH+glAh/TYAbSPVc9jNyzSkVHkH9Trc348Yf0GogKqG+OvHG3LbMi0JE8Pu36bM08EwIMZfECqlvxagLdk+A0UZ9rx7X6bm/zAgxl8QKqTf9tnQ/Pt7aKEEZap6SraPIAh99/7Mwm79PbZQTHLLS3j+w/79iPEXhAoolRLYBW1D9O/roYUSpLX6Rx8x/oJQIX2XfSTbZ6CUCeRLto8gzGCqyvgIQ8ZYLSoaMewBxelIWrenuKTzsH8/YvwFoUKqkH1qQWT8h92znI6UyeQJK5L8+o0Yf0GokP57/kA9iH62w21apicdzeE75F+QGH9BqJAqUj1Tz7+/xxaK6WgO38pb0xti/AWhAirL9gkZdWX8h968TD/KSG2S6ikIQl8NgMrxV56/2P6pp0x5h3Sqx+H+gsT4C0KF9PPnr3L8x2rRz3bYs0mmJ+XLOwy57a/O+BPRZ4noISK6l4iuJKI52roPE9EqInqYiF5ZVRsEYdD0s/5O2/L8RfaZesqVd1DbDPf3U6Xnfx2AZzPzcwA8AuDDAEBEiwFcCOA0AOcD+DIR1SpshyBMOf32/prtMOlI6hLwHRilBnnN9KqezPxLZm7FH28FMC9+vxTAD5h5kpnXAFgF4Kyq2iEI/Wbjzv1Ytmabsex7t63Fmq178csVm/CTO9cly392zwbct25nT+dbtmYbnv3xa7F1dwNA6vm3Q8ZP71qPdsi4YvkTeOTJ3c79dx1o4ou/Xpk8OTy6ZQ++v2xtsv7WR5/CdQ88iS/+eiV2H2j21FadBzbswpV3rSveUOO+dTtx1T0bsNtqMwDcv35nz/cSADbs2I9v3LIGAPDUnkl89cbVYGZ87aZHsXn3Aec+T2zbh2///rHcQP7O/U185hcPodWO1q3Zuhffve1x5/HuWrsdV9+7sedr6YX6FJ3nLwD8MH5/AqLOQLEuXmZARBcBuAgAFixYUHX7BKE0537uRuxrtPHYp18NIArEfuTK+3D4QWN43oI52LpnMtn28t89hst/91iybTc8sW0fJlshnowNkzL+d67djkt+eDeOn3MQPnHVCrzlrAX4n69ZnNn/lpVb8U+/fASvWHwsnnHsoXj1F27B/mYbbzkr+l1ddtOjuGXlVjTaIU6eOxuvOv24rtuq86ov3AwAeMPz5hVsmfLaL94CAPhvL1yA79221mjPa/41WtfLvQSAa+7biH+4+kH86fPn4VcPbsan/+shnH3y03DpNQ/i4Ika/uyFJ2b2+dm9G/CZXzyMf3rTcwG4vfrfr34KX75hNU6eOxsAsPzx7Vj++Hb8t7MWgIiMbb/9+8dxx+Pb8ern9Oded0NPnj8RXU9E9zv+lmrbfBRAC8B3Ozk2M1/GzEuYecncuXN7aaYg9JV9jbbxuRmGAIDdB5poh9z3x/1mOzp+oxW9KuM/2UqXt0NOtrNR3nMrbuf+ptn+VshoxPsOyxwBeycj0WCy1S7YsnPU/WiHnATRW8n1u/cJk32i7VyZPG1rG3u5Titk5/KppCfPn5nPy1tPRH8O4DUAzuX0bq0HMF/bbF68TBBGkmb8mF+vBQi5AuMfGwll/IPYi1QGiRGd02f8VXMsmwRmBhEZZaKHJYOIijfpGmXw9Y66XZDDqRa31D13bKaOq/4f9OW2oQ2HwPhXme1zPoAPAngdM+/TVl0F4EIimiCikwAsArCsqnYIQtU0Y6NcDwjtkPvuPavjKy84CMzCbiFHHnuj5TNcsbGz2pUxfBi+3PQqmqM6u5DT70rdG6/nb90r12ah9kRhLs9uqz91DIoqNf8vApgAcF2sd93KzO9m5hVEdAWABxDJQRczc/+f7QRhilAedz2g5IfeTyOqjj+ZeP7RcmU8ODZiLZeVgdvIA6kB043QsMg+tkbeT9QDUjvkzFOR73tTWyqv3un5J/Ja1vPPbMtsPHENgsqMPzM/PWfdpQAurercgjCVKL28XguiH3W/PX/L+NfIrOrJHHmmPtknZDZe9eU1WLKP+xADo4q+yJR9OvX8Y83f4funxy3W/MMh8PxlhK8g9Eii+ceyT79/0422pflrqZ5AatT9sg+M7TPLh9Hzr/DYuuyTlGjOkXOiFaZX77pNocfzd3n4bZ7Gmr8gzBQM2adCz79hef5tS2Mu9Pxto+RYXqXG3g1VmEfd8w/ZvIc+2Sfx/Nv+7dLMoRKyTzh42UeMvyD0iDLK9VqQ6Mj9/FmrgK+Sl+Jy/hmPvijbxyczVO35dyVvVOj6uzz/NH7i3kfJPK2cJ4ROPP+QRfYRhJFFeX+651+F7KOMyWTTTPVMPFguMP6Wd6tIvV59WZ8ardGLvFFF9lH6xJR9+vF1fuoSWkmef95xLc3f6/l33vZ+IsZfELpE/abTPH+Kvcn+dgDK42+041RPledvGfVG233SRNf2pHqaef79N7bdHJMqdP3bjs4w8fw9+2Ty/J0BX3ObZLkz4NvlE1EfEeMvCF0SWh53LQhiHbm/50ny/JvmCF87r1xtl20n4u3s5eaTA1Ctp90Nas9+6uPOPP8Czz95espJ9Uy+D6sTdub5S8BXEEYX9dtVnvlYjRBy/0v5JgHftiX7xEZFeZp+zd8t+7gMaxX2qBt5w07zb/ZRI3GN8FVG36/5R7SSTsJ/3FJ5/uo4A+wAxPgLQpcknr9Wc6cKLbdppXrGc7mkuemF2T5mexWsBjsNY8BXYUlr/aCtXbf9VOTN9rGespyyj1VDyV5uHM8z6noqEeMvCF1ia/5jsewD9Nf7b1ipnoE1yCv1/H1uq8/zz0odwxLwVY5/MrLWI2l1Q6h53ck9yMnfj9oRkXj1ebKPJ7CuY6fpDoKpKuksCNMOW/NXAd9+ky3v4Bnk1aHnnyzXdqtC8+/HPfE91XRD6JB9ikf4mpk8rs28ef6uQV4FMYapQDx/QegS2+jWqkr1jI3JpDXCN0k/bBdo/j7P37G8EtmnG89fzVFvxVX60h5NcmE2jbDviS2T7eO4Tz7PP1f2Ec1fEEYP9bttJgHfwDvCtxePumEFfGukzm8aLtvjTM4dv5Yr7NZ1M710J/uYEd9+av6p7KM//eR7/op2jjxkj7tIzpcb8C3d7L4jxl8QuiQZ5GUFfBlZ49CLh5eWdzBLOttVJDuXfbIZJ8OS569Qe/ZT9lH9SJvL1/NPOti8bB+VfVVC9gm1NgwKMf6C0CWp5x8HfGtK9sn+oHvxXO3aPrbmr5d3cJ07lTbsFdGLmeffdTO99EPaaFQU8E2zbuJ1+THzNM/fIQ+lHUToXK4jmr8gjDBZzT+I8vw5GxDsRbNutkzN3y7prEsRLkOb8W6T9meXV5F33pPnb0lr/UDvNFXL9FnRXNiGPb+8g/s+u7aVPH9BGEEy2T6a7GPT6sX4256/leevGxvXE4ae0mnM2oWsAapG8+98H/su9jXP3xHwLcr2UYvtAVzGcS0Zzl6uI3n+gjDKOLxS32QuvRivhjfVMz6nZlxcTxi6h6+31aU7D0u2j2pGOoNWRXn+1tOPN88/85SVY9Btzz8n1VOyfQRhBLE1f2Y2DIpOL8arlRnha9X20U7oOg97jH8SCxjCPP9+ymY2uudvB7290zjGi9V3kSfllKnnHzru/VQjxl8QuiSdQSuMP6cGxf6996T5e2r7uDxNl/HXU0L1JxB7gBNQzeQpXRl/Nl/7OcI3DbZm0119Lc1M+tJBwFeyfQRhmmFr/onRd/yefTn4ZfCVd3Bq/p6pHKPtYHn+avkQyj4Vav66t68ut6zn3ywR8M1M5jKk5R3E+AtCl9iZKHnpe73IPml5hyjPv+aZyQvwaP5au/SUyZCzUwlWYYu66lDYeKku26dkwNfOjHJt5orB6MuN40mqpyCMLqnnb766fs69yT7m8X15/tE2Ds0/fs1o/ig3GrVXurl0ux391PyVKtNm1u5N9OoN+KppHHPq+SeF9srU9pHyDoIwuqjfbSPx/FP5x+4CupV92mF20o9khK816hTI1/wj429KPNl6P101M5fuZB+T/o7wTWUf2wP3T+YS75tcS3mDLrKPIEwz7Hr+adGv7LbdGi/XfmqQl56ymG6fPXmia7Od7eMo+dBHY0RWDaJOsHfpJWZikxhew/PPP35aPjvu4F2zc4XmNvZynaLOZiqo3PgT0fuJiInoqPgzEdEXiGgVEd1LRGdW3QZBqAJ7Anc9V7xf2T6uQUVpVc9ynj8bnr+Z6llmNGq32PJUJyR7WPdYpbn2gh7wLTuNo7qEvDl8Xd+H75jT3vMnovkA/hjAWm3xBQAWxX8XAfhKlW0QpgeTrTb2NVqDboaBneffyqn70q3n6kpxVPZP2XHduORq/naqJ7IebD89UWWmu0lnVB2W2rPRR+NvBnyjZaWncczR/L15/hlpLZ1BbDp7/v8M4IMwBbKlAL7NEbcCmENEx1XcDmHE+X+veQjv+ObtlRz7Kzesxjd/u6bj/ezaPokk0EfZx9Uuu7ZPWc0/zIzw5YxhtlMdv/W7x3Dtik24f/3Oju9RYMlTnaD2uPx3j+EvLr8dV929AUBUQkPx9VvW4Bf3b8SXfrMqd3Daxp37ce7nbsBHrrwPgC77ZEfu+j1/czt7q2tXbMIdj283tlHkPV0V/Vt84Vcr8en/eih/oy6pbCYvIloKYD0z30PmbMwnAHhC+7wuXrbR2v8iRE8GWLBgQVXNFEaEzbsPYMueyUqO/YsVmzB7ooZ3/OFJHe0XJhkiphF2V/XszvhfsXwdAOCM+XNw9xM7AGQDvrpxbTjy/NNgpdlRMBcXIfvW7x7DomNm49oVTwJAZ/coeULpwruNd3l0y148ufMAjpw9rh8SAPAv1z+C3Qeip8GXPWMuTjv+cOeh1mzZi9Xx3/963Wlmnn+8Tchuo263xxfXedd37kjeF8k++v0ouje3P7YNeyareeLtyfMnouuJ6H7H31IAHwHwsW6PzcyXMfMSZl4yd+7cXpopTAPCsJpMlOjYWe271H5Wbrg+9N8+WreDlAIC3vT8eXjuvNSw1YLOUj11icGcrzdbh8iVYtmtZBX0EvDV7uCLnn4Ubv7gy/GXLznJuK9Fge5kO20Vwwz4Zjz6Qs8/zN1O3yb97D6W/d4Fc/oE1W968vyZ+TzXciI6HcBJAJTXPw/AnUR0FoD1AOZrm8+LlwmCF98MWf2gHXJXNVZsvdiomNmn/Pkw/vHrT8+2US0M+ELXuDvz/Ftt7jpYrWbj6qqqp9YOJXMRmXMk65JVXued7fDSffSnIvu8rvakAV8/RbV9OvH8Q2b0IczhpBLNn5nvY+ajmXkhMy9EJO2cycybAFwF4G1x1s/ZAHYy88a84wlCyNUFx1zad9n9otfoc1PrQeyjdVswLWRGEJjeX6qlm+0ACgq7MVtF3LL31G5nsx12LVklgekeUz3Vkw6RuVy/lrz/jdDT4UXZPuY2RZp/Syvi5yMj+9gxAE8H5jsvDaPn3yXXAHgVgFUA9gF4xwDaIIwYzN1552VwDaQqQ5oh4vL87W27a1vIkcere392bR/d2DScef66sdPaiOw9dck+xsCwkJOYQxHUU8A33Uedj0DGfS3r+RsdhpbeGu1vyz6+9pjnybuiwoBvyM73LqInv9xNumZKjH/s/av3DODiqTivMH2IiqZVJPt0KSnZXmPLGj1rbttd2zl+7NcNrl3S2dC+HamhuswRsrk8W97B3Nf2/NvMCFDW+Kfn7RRT9oleAzI7hbIF6UzZx8rz10o9qPXu9rCxXd7XWTSNYyeyDzMjCKpJypQRvsJIEP1oKzp22J3xt+vh6z/6/nn+HGv+6bJseYf0vEWyj6n5uwZ52bKPWQyuE0NuP6F0S6DJPkmHm/Gm/fubHZ7p+avOJB1X4G5rGhso9vztW5R5Eugg4KtiPlUgxl8YCSoN+DJ3WXxMvRZ7/t1r/tGPP3AGfONXre15Ad/Q8vzZ2c70vZLD7LEBZUnKO/QywhdpwDcgynjgSVvLav6h/iRkPhUBxbKP65hFZJ6utK+o6P8u0vxLn6ojxPgLIwFX6vl3Z6DsUaG69t6vmjnqx6/rvnZtH9Pzd2n+0WubLUNY4Pkro68fc6o8f2fAF1ljnbbLb0XZ8rT14G4228fd1oyx7+CSegv4iucvzHAq1fzDXrN9lOfff9mHnZ6/NZOXduy82j6ZgC/np3qmxl/z/Dt4QlIt7srz19qZyj7pyGaXPOXDK/toqa925la2PdbnDqx/Js+/g4CvivlUgRh/YSSoWvbpzkBFr658+6zx797zDwhmnr81yMsc7JQf8DUzX/ID0+k8AmbAtyyqzd0EfPXzpHn+0WdXp5WXjpoJ+OrGX52vQMvPk8eK6D3PXzx/YQZTdcC3F8+/jOzTbb+lfvyG7BOY5zBH+DpkH5XOyLbn76rnn+1Iug34Jtk+XVy7fh4l+ygjyMg+geQb//Q9M2uZPaYEpL8W0cn32ZPsE6KyPH8x/sJI4HrU7xdthxEsgx3wzUs97GWEL3lkH1d5B+c0jqqdjlTPbGwifd+w5ino9DqCHgK++nnU9ZK2zjaaeXMXm5q/u6pncg89h+nl+8wGfMvLPiM3wlcQ+k3I1dX2ico79KL5Z9fZi7rX/OM8f8cgLz2FU+HK89e3swd55dWdcWn+HXn+6D7ga3r+0auSu1yyT14JClPqMr39svX88+K9RbGoXjx/FfOpAjH+wkhQpebfrexjBwvNddY5evD87do+NSvPv3AOX227Tmr72HMH2+cqIp13oIuOVbsMe0Sx63+hvOZvBXytbXxfU56MV3R5eSN+y3j+VaV6DqK8gyB0jEui6Bdd5/kXFAPT6am2D9m1fdJ1QAnNX5OnTP0bCGEbtfyOpLM8/+7LO7gCvvo96Czgq7238vzL1vPPLi7fIebn+ZeRfaZPbR9B6Bi2DFc/iQxCL7JP8b7dtJ050qR9tX2S0gTxwcdq5NH8Odk+U0445wnFdazuAr69yT6Ble2je++K/FTPdJ0+JsLI88+X/LOyjyUl5ZEn+xTdTgbE8xdmNlXX9ulHwDd/225kpejVzvP31fOfqNcKp3EsGuGrr3fV8e+qkyx5b+3BWIogyfZR22Xb0XDEOlzH1YPXbW3cg2qj73/MzuvXPxXdk7xUz6J9RfMXZjz643q/6Tbgm2r+xdt203ZlGKI8/3R5traPMv5B4TSOmdGuJUb46nQij7kC0mW2j86TlX1UANmWr3xtVbgGrql22dk+fs3f/tyB7JMTVJ929fwFod9UFfBNBvx0JftEr2WeSLp5alHHDwJ7MhdzDl91XybqgTvlUTPCoeX6Z/P80/d5ElIZUj293PY+o6iyfZJBXsi2266h7ztuyypPbT+9+TV//30qGvWckX30p48iyahCzV+MvzASMKu//nYAroyZshSVBHBt283xvbV94kOqshITY7UCzd/O88+fxtGVNtrJfepEFtO3t/fJlHdwxGjyZB9DytIsdXQt5vfva6l92frHIgPeU56/DPISZjpFqXjdUpTlkUf1Ad/o1VfVM2179Hm8FmRqyQO25p82JEr1tNupyz69af5sGdYivJ4/WZo/XAHfcpp/06q8mmT+FHTk9ovVoZ8AACAASURBVGL2tNWFfc/MSWhyd5XaPoLQ6fD7To/by4QjZeIFvXQumUFe1mQuyuBPjLllHzO10Ty+7ZXqH92af+eef9l9zLal7/WqnmpdR6me2rYta8yCXvQuakM52cc4fqfZPh3JPhLwFWY4iaGtzPPvXFLq5Gmkmz4rNf75g7yUs+8L+Pqnccwapv5q/p3t4/P8lfFLR/h2X9VTn2c5KnGt3ud/l/byjgK+OfuWKu9QkZUW4y+MBJy89tnzNyYB73DfDoxbd+UjoldfbR913sTzr7s1fz3rxhzkVa6ev05n2T59kn1KeP555R3yAr6cvI/b7Pn/yhvhWyj7ZDx/7X0Jz180f2FGU5nm34EHZ9NZwLejQwNIjWe2tk98zNA8ttfzN2byMjX/KgO+iWHtU8BXpfswHJ5/bp5/+l6fc0GXfdLy2MXHsD8X5vnnyD5Sz18QCqhK8+9kwI1NXm0fm+40/+jVN8jLPubEWOAO0mqTlNsVLvMGefUa8O00nuILomYCvpx9Ailb26dp6e36UxHQgeffS3mHDoPFovkLMxrby+3bcbl745/m+Rdv212ef+r5G4O8PNMjTtRr7qqeWtaNmeafrWmkt7PX8g6JYS2d56+/12Wf6FUN8mKH7FNW89c9f/1JSHngZS+vE89fAr6C0AOdeNmd0MmsSjZVp3qmef5uz98+7UQ9KFHP3/T8s15p+t6p+Xfh+fct4Kulemby/LvQ/F2ef9l6/vqnos4tz/MfZFVPMf7CSJB42V1U38zD1F8727ejgG8XnZbaJSAyMj58nqA/2yd6dQV8Oy3v0FHgOvH8ezP+ScA3KezWWaqnkecfuj3/juv5d5Lt04Pnz6Pq+RPRXxHRQ0S0gog+oy3/MBGtIqKHieiVVbZBmB5UnecPdF7iQRmAMnv15vnD8vzd24/XA6dh1MssdFPPX6ezPP/4vCXvqymlpO9r9ghfdszkVbK2T8bzV9sUjvC176l/XdG+5tNm7q6osrZPZVU9ieiPACwF8FxmniSio+PliwFcCOA0AMcDuJ6ITmXmdlVtEUafTksFlEX/If76oc246p4N+NY7XlAqvU4VSqtK8089fzPdz9e2iXoNQGQIa0EtPY5qL2dn8soaNU3zd8QPOhvhmz1mHr74iz2NI3P2CSRvGkdD9rGf9JL/q/y22ktN2aczz9/oOAYY8K2ypPN7AHyamScBgJk3x8uXAvhBvHwNEa0CcBaA31fYFmHE6aSCJgDsPtDEQ5t247nz5uCTP1uBS847FUceMo5P/fwBvPMlJ2HeEQdj+94GPvST+5J97nh8O256ZAsa7TAxpO2Q8dlrH8Y7X3ISjpo9YZxj294G/vaKe0q1Rxmgp/ZM4nPXPYKPv3YxGq0Q/3jNQ/j7Vz8Lh0yYP8UtuyfxtZsfBeCv528zUY8eCb5642pcct6pyXI9nVG/f/983Uo8Z97hznYCcJaKUJ7qz+/dgGY7xBueNw/XPfAkntoziQvPWmBsqw51zX2bcO2KTWi2QzyyaTeOn3MQfrFiE8591jF469knaufWz5OVfdR1f/oXD+GPFx9jnGt/s433X3EPagFw6RtOx5j2eOQK+NYCwgMbd6Xnixt759odeNs3lqFGwCtPOxabdh2I7qVD9mm2Q3zwR/fizrXbM/dJZ1+jjQ//5D6cuWAO9jfb+IerH4yvJzrvrY8+hXue2IGn9jZw6jGHYsOO/Wi2Q5z/7GMrzfOv0vifCuAlRHQpgAMAPsDMtwM4AcCt2nbr4mUGRHQRgIsAYMGCBfZqYYaR5vmXs/4/vmMdPnX1g/jJe16E7962Fi88+Wk4c8EcXP67x/DMYw/FhWctwLLHtmHZmm3JPko6aLUZyhav2boXX71xNZ557KF4/fPMf9M71+7ALau2lmx/9PqP1zyEH9+5DktOPAJzD53A95etxevPOB4vPPlpxvbv+8Fd+N3qpwA4Uj09xuBFTz8Kn7vuEfzsng2G8fcFfNdu24f1O/Y72wmo+xBgUnsCUEbye7etxYFmG2943jxcsfwJrH1qX8b46+d613fuyLR3084DpvH3pN3ak7lcfe9GHDbLNF3rtu/D2m37onOdcwpOmTs7Weeq5/+q04/Dz+7Z4Dz3zSu3gBm4aeVWhMy45LxTnQHfDTv248q71ifLjj98FhYdcyhufGSLse2KDbtw77qd+OHta0FEaIeMQyfqmD2rjjBk/OyeDfjubWsz96cW0PDm+RPR9UR0v+NvKaKO5UgAZwP4fwBcQR10Ycx8GTMvYeYlc+fO7aWZwjQglX3Kbb+v2UY75MRwtdqhke8OZHVi14TlebnqeTpztv3mOYPYCOjt0dm2t5G8z9b2cZ/j9BMOx2uec5x3snE9u0WRN8K3HTLGrQBDUgI75KRsgW8OZAbwzhef5G4s8gdO5Xn+ANBo2e3OtjFdpwV84/v/p2eegKMPTZ/k9PbPHq8nbVBxEfvqXLGVz77puXjmcYfCRi8hot5/5o3PwXg9yBTbM84RP6kNpezDzOf51hHRewD8hKNudxkRhQCOArAewHxt03nxMkHw0nHaYGga22Y71Grh5Bt/PW0wz0Dn1ZC3sVMKg4C0HPPs9naqo6u2j01A0bpssTZN9ikcUaq1gRlj9QCYzLYr1DKFMvMEaOcdq/v9y6Z14XYaqsKu5w/kd7x56asq4FsLyLiP+i72NTfbYfb/zmG0AyLvU5lNEETbtkP/LHKqYxhKz7+AnwL4IwAgolMBjAPYCuAqABcS0QQRnQRgEYBlFbZDmAao31lZ469sQyMx6OmPTL3aQULlTTatyo+AOzDXifG3BxNFP/z4HI5r0q/TzvbxeYLK+PgCjHoJ46J2qraO1cxz6fMf6PfT6flz1CHZTw8K24D7vttU9knbkmv88zz/uMOpWVKajn3NjXbofJqym2B3KHnUiBIHwHcpqgrrKGr+3wDwDSK6H0ADwNvjp4AVRHQFgAcAtABcLJk+QhGd1vZJpJ1Y9mm2Uu9N2XZ7YFDylNByyD4uz78j2SdulzL+QX7Hoi8KSgZ8ieInCo/x0+es9bdT6/gYRuBUb2tUETN9CnB5ryEzCBRPLJ89l935+jomu7AbUFTC2fysX7Ly/IMcQ123dLVmy2H8HbJPLSgv0dQ0z9/3nbQ0ibAKKjP+zNwA8H971l0K4NKqzi1MP7qXfVKJxza2tgFR2S1Nh+zj9PxzSgoUtT8gSp4cXIbTln1cI3x1KE4HrZFL9lFtKFOKQH+f1fx16Uz3/J2yDyLPf6wewGX97fvvC+antX00zT/n3ufNnKUbVK98FsTymfY/lJnG0ZEma39PeQRB5Pm3Q3+qqPr/GEXZRxD6RqcBXzuoqxt/b8A39kT1J4LcgG8HQ4Lt6pG6cXE9VZjG39S7XbqyXvM+b3auIqnKHrlatyQQM+Cbxix8sg+IMk8PCvvJy9e0dBrHdFleFc/cgWvak5fPqNYs7T7S/M1tQofRjmQfb7Oy5wjipyZPp5ckB1Rk/cX4CyNBp7V9lJHSNX/bkNsjWNW25mxP8avjtB15/tZxzIBvkeZvepTk+NUq+6AMio7+qUiqMnLt4wFGuu3xBXx9BeL6ofmn2T7adeR0vHllqst5/mY5jUY7zFT7ZGQ7vCDW8csQBGnGly8Ir/6/pLaPMKMpGoFpkxr4rOefdAyW9+hK9bT3YYchAYofzV2GvhPPvyjPXwUF3QFfvc352SN2wNcOYqrOS/f8Xbq1aoLS/F3Yna/vq02vvaTsU3D9QNSh+CSaMp6/qy6S0vHLoALOvngJkD5ZjmRtH0HoF51MnAIUBXw9so8j1dPeRzdQem14O0ioUw8oE/C1NXMbe5BTUcBXrXcFfO0257XV9vxtI2mmd6Zt9RncgLJB4+T4VppjJ55/nuyTDXhr+yU6ut/zrwWmB99suUt4uGWfDgK+QVHAVzR/YYaj18/pLeBrr7ONf1YOsg20S0IA/AOvonWUav4Ob9l1TYbnH9i1fRzn0D3/Atkjr6225m8bSf0JJi/gm3j+BNRzhHD9O/B9tcqbLp3q6QjOKvTyDv7xEua6KNUz26E6ZZ8+5vm3xPMXZjr6b6xsjFX9MJ15/pw18kAqAzUd5QxcVR91zb/Y8zdlIz2/22XHdINQprZPYvwDh+yjvW+2izx/0xO3pYy21okZOf8eg0tEGPfIPlF7sk9ZNur0huffUZ6/vl8J2SfIyj52yxic+V9Ug+zKUIvLdEcBX/c2zXZ6D6tAjL8w9HQz25ayDbqOnwZ8zXUKZ3kHq8NwDRgC8h/Na5SVfdphtmPRyeT5B/mef2IgtY7GdaxWGGbaqn82ZJ+Qk8FIyfpEttLeO2WftF0+2QcwO+Ai2ccc4ev/P8irVKp70/kBX1vzz95Tu8Oz5aI8DNnHG/BVbS11yI4R4y8MPfpvo+wgL2d5B0u68Rp/R1A2Dfim2xuef46Bq9VSg5wETK1sGZtswDdd57IFhuyTjU4ax7XbWjcqYGrGOExz3tP90+PkBXzVxyAn1ROwplX0yT5WPX8gf+auTKqr9tko7+AN+CLr+WduadZod1TeIQn4lsnzF89fmKF05flb0o5e28eX6plsmzPC1zD+1kAsHzWiNGZRMuCrdwh2bR+XDGAEfNn0dk3ZI8y0dcxT48YV8DVkn6Qz9ZdUICC3vk/D0Pzd361dz19dh4/c8g5K8yfyxj7seECj5ejcHOfpKM8/PkepPH/x/IWZiqH5d5rnr3T8tuZpW2MAFO5UT/M1W98xop7zC61pUowRJM0J+OpepV3bp8jzj46ZrjMDnpxpq8/zTwO+2XaFoXk/7Q5HvQsKNX93J6XjquqZm+3jkGjUJSfedODX5+1pM1th6JzNxSn7lC7vkD6lFef5i+cvzFB81R7zyBvhm8g+lgFRhsEw/paBLjJQvnW2odc95zKevyH7ODV/FfBF5pi67NGMc/d19Dz8bgO+0fbIHMen+atzlgn4ptk+2nVonUamBIUj7VRds57t00nA1+X5O2WfsoO8SJV3yPH8RfYRZjrZn10xrkFe2RG+oXMffQCR/bTgrT9TaPzNc+g18F3GXz+NnUJYJPsApiHVj+5K9dSzf8oGfCOPVe2TvY404OvW/A+Oa+Y3HBJb5toCdY1uzd8eROYa4ZsY/7iNtU7z/K1tXHJNx3n+8SAvCfgKgoduPP80qKu8/DS10l5n4xrh6/JudYqMfybPX/vRF0lZdm0fIPvZln1MQ2xKOXZQUq/fk8nzt0f45nReRoeja/4O43/IeDrfcLpPZrPomlTA1706E1NwpXqqDs6o6unxqO3AbcPl+TviHHllop3nKPD800Fe4vkLMxS2Kk2Wwdb1G46Ary9jpJUj+/gePMp6/mbAV7U1/1rs2j5A1hCmtX3IaDdgjfBthw7Zx+35h8yZUgfugK/f8w8IGK9n783B8TyZpVI91fk9t9juXFwBX3XNRj1/r+ZvxRec9fwdVT1z4gg26ukiCpa7t1Ftldo+woylK81f5fK3NNlHy7FXy1zoso9djdMX8M1L8VOP94A5wtc1dsCFbYyArPSjPqvtfPPhthyav2sEr2qjLYHoef7qs6tGUar5u2Ufl+fv+27V+X0ecGaqSYeXroLcZTz/jOzjTPXMGu3OAr6EGrlHRyvE8xdmPC45oew+neT5K1zZPmUDvi7Pz/T84+NyNp7gww74Rsusz4HZDsML15a32oyaJfrr2T9mZlXWSLbtTkx/ArDOqdqZq/l3EvB1rkW27LR1GEPzL13eIf3cbDMYZqfJjMoDvonmX5GVFuMvDD0hu9/nYefyN9tZjdqeSUrhKu9QNuDrSvk0NP+SAV8dO88fiKpl2tsAqZdse+GqXZHsYx7fNs56W+14gz7IS73awWx1zqihHs9/Ivb8W7rm774PiefvMaxlZJ/k+rWAr+94drZPoxXGcQNze3fA13lI7znyAr6S7SPMeNghJxShtks0/1aYCbCW8fxtqahImvAZ/4zso/3oi55mojx/e6F1fjvPX780XfYoCPgCMIy5nRWj5/dHbfc8FSnbj2w2DpB6/obm79G+CwO+BbJPyNEoa3VNgKqn7z6eq7wDszk+wlXSwhXw9Z6D0jpMRZ6/5PkLMxZX/ngRznr+lsbuC/jqmr8tFRUN8nKVeagFZGjk6rh59fx1XNUiM31BEvBF5pistavZDjPGZCxwG8+QHdk+VoqjK/Crzqnanuv5d5Tn79P8LY/ckeevsn30Eb7+8g7uwm76d6t3egrXvMBF9YPy6vlLSWdhxmNq/uX2ceb5Wxq7byYul0FKZR/3+ZSxcHr+joAvsy6X5F+LmvVJx5fq6Qv46gFPV7xAP55u/G1vNgzNLBd9pLJL9vEN8nJr/pnNkva5rllhd7iuVE895gHkF2HLBnzZCBoDqqpntsF2h+IdS0DaBO5e2UdKOgsznG5q+6SjeDXNv2S2j5HqGdpG232+RPN3SByR5m8dLzSNrI5tvCLN3zxmVvM322GnXap2RVU9swZKX8ZapxQE1jSObJYy9tUo0lM93bJP5Pm3HJlVNq4J3HXKDPJKZa/QmOzehZ3nPxnHJczxEHCWYrY7lLyMovJ5/s7VPSPGXxh67AyUMtief5Tnr46RL/sYk7koA10wICsN+HpkH8s7zivvYHdKAWWNSjb7x8w2ygZ8lezBDl3a7EoMz9+aSMY2Vm3tCcbVSRMI447Cbqnm30FVT/dqR8DXXK97/vr1+z1/02OfbLUBmN8twz3mxNWxujCrejo30QoHiucvzFC6q+oZvTZyZB9/nr8r4Ks0fzdFnr9qt/Lm8rJ97E7JnsxFLXN9dsk+uucfbZttnyHtWAFfu2PQ29tyxEfUOdW5etX87WuzKc7zN4O1taSjdB4uI/s0nJ6/22O3j5lXQqIWmLKZD/H8hRmLbhtL5/lbBp45NaqdlXdQbTCfAGyK8vyZo7arNuSVd7ALzkWeeX7AN3+Er5mjnvFOLVlJv9YgICvV09SofcY7Nf4FtX1KGH+FT/ruZIQvoI2JyJV90s/K+Nt5/q5Areupyn2O6Mmjzf6Ab9ExeqUy409EZxDRrUR0NxEtJ6Kz4uVERF8golVEdC8RnVlVG4TpQXcjfLMGfrLZTtblzZ3qqupZ9ANVhsHOnAHSaRz1+v959fztTika4WsdNPMkoLZ1aP4wg5V2EwNb808GoinPX5d9zI6l4egoo3OmenWe5q+PtSjq173Gv6i2T2ga7lqh7JOdwxewA75uR6BMtk+gxRzs1FkXozjI6zMAPsnMZwD4WPwZAC4AsCj+uwjAVypsgzAN6CXPX/eiD8Tv2yHnTgaiG6R0tq3886dSguvHHmm7l//2sWRZXnmHrObvGuSVPYd+fiMoy+a8vS7PX2+2Hp+oBWSczB6U5CqCF20Xt5OysgwAzBoLUAuolOyjsJ9+FHkB34c37caqLXuMoLky+nmev36PJpvK+Oupnm7Zp4zmrz8l2qmzLkYxz58BHBa/PxzAhvj9UgDf5ohbAcwhouMqbMeMhZnx/ivuwe9XPzXopnTEgWYbX/rNKuzc3wRgevvv/d5dWL1lD25euQUPbNiVLH9qzyTe/Z07kn1cxdsOxJ5/yPnG/6m9k3jzv/0eb/36bcl2Ycj4l+sfwTmfvcG5jxpE5NL86zXC2m37cOk1DybLjGkcQ8a/3bga96/fCQB413fuMPZ3DfKyvdZMPX9mfPO3a7DkH67Hk7smLc3fne2TdBxWnr++tR3w/avv3ZW8V/t9f9la/HbV1qidHtmnHgQYq9nGP7OZdc3W57hheudSCwiX3fQofnZPZG7+/qf3YcvuScOg50l0arnT89c1f/Ti+acxDGZzRjgXIyf7ALgEwGeJ6AkA/wTgw/HyEwA8oW23Ll5mQEQXxXLR8i1btlTYzOlLK2T8+M51+P3qrYNuSkfcuXY7Pnvtw3jzV3+PS35wF66+d6Ox/o7Ht+PjV63AV25cnSy7d/1O/GLFJjy0MeoQ7MlcgNT4P/bUPtz0SHRPPv7axXjoU+cbx1+9ZS+WrdmGm1duxba9jeh4IeNrNz3qbXNenr/68U7UA7zp+fMAqIAv4mMDn7n24cRgPbhpV2b/okFeyi7pss+963Zi657JTLtcHcnbX7QQrzztmGRf9WrHA2y5bOXmPcn7VZv34P1X3IPPX78SP7w9/YmfdfKROO9ZRyefzz/tWDz/xCMwVguMzrlldcjPOu4wfOw1i7VrNht+8tzZeNXp0bGS+xAQJlthci/3Trbja9TiIiVkH31dmu1jaf5xLOE/3v0H+NTrn53sax/LdXz9Vb/uP3/Rwsz2QxnwJaLrieh+x99SAO8B8DfMPB/A3wD4eifHZubLmHkJMy+ZO3duL82csaRpjiWF8iFBGZeHn9yNq+7ZgFsfNZ9cmu0Qk80w0fABvXqnnZqZ7ncgfnzfsnsSF3/vTgDARL2GWWM17zSMSioK2Te2NyIv1VMd+5jDZuGzb3oujjxk3PD8W2EYS1HR5/FagIteenKyf+T5uz19hUqn1L13vePLl32AD7zyGXjZqZGBboacDEKLPH/TC/aVYbh55Vb8+M512LTrQBIkDYhw2KwxfPy1pyXbfXLpaThkoo7xWmC00X4aO2P+HPzFi0/S2m2e77BZdXz5z56P+UcenFy7MuzqWC1toBRZRr90Pf9WVvaJsp6iY7xg4ZF469knJvvquM5hS4T6/+iFZ813tqcK6r3szMzn+dYR0bcBvC/++B8A/k/8fj0A/QrnxcuEPpMOcCoYQjpk2FLApJX90myFaLZDy3CY1+rSUQ9onYVCPcqP1QK0wux6XSrKQ/1AXbKPMjZKmw6IosBp/KtX50gNFhtSRkAEsvoU+yxKWtEnczGMv9Yun4Q0Ftfdb8aFzNTxdNvT0spk2BxopfdPecu2zq6uR7VZj6/Yge7MwDbrc3LNmjHX8/n1Vz1dttjzt/P8s7IPWMli9r7mMV3nyCtR7ZLIKrL9lco+GwCcE79/OYCV8furALwtzvo5G8BOZt7oOoDQG3qO+yhhG4GM8W9zbPyzgcckndNxyQcck34rI+vKSAFSw1yU7VO3HuVd61JjZeb5qycSVXa6HbJhBNyev/s6lHcbak8Sdrsymj+Z7dPLX9es0g/6SGkb/UlM9/z1c+htGatTrudfNLZBPe2kg7bSffSCfupYGc3fN/rWktmcqZ5IZTF7Xx1fob/oNXtuV3B8KD3/Av4SwOeJqA7gAKLMHgC4BsCrAKwCsA/AOypsw4ymObLG32zvZMv0yBux4df1YrujcxmoSYfnrwyeaxRqtI96ksjvAALLwOuoH3sizRAZso/qYBra08yYNvuVq56/7fvbXnA7I/v4Pf+a1XZ92kJb9nFNZq5QnRiQdthqT91D1jubRo7xLxrbkOf5t6z/Bz1uks594LyMuEBb+jnx/A3Nn5NpLs19rWM5DLddh8l1TUXH6AeVGX9mvgXA8x3LGcDFVZ1XSEm8YU/d+mElY/ybtucfGoZS3ydX9nF4/mM1v9GO9olln4L00DzP3zauQUCm56/FK9Q5bNmnqFTwWD19qgCysk8tR/NXBmw88fxT796WfRraU4HNAYfnTw7PXzXF1vzt2FRGniK7wzOPrVfVtGXAKLceme1d2JOypJ6/Vd6B82dFc32OluWtc3UWzmb2jIzwncaMrOffKpJ9spp/y9J4XWl4Ts+/rmQfj/HX9Hj9kLZMlDuZS+Lpph2EPrJTtSuKZXCmPbrhUmT1b9ObjAK+nFmvb5Ntnyb7sJJ9zG2becbf0PyV8UfmOHpn2LKku7GclFSf5q/PYqauLYmfJMfXPf982Ydg3iP15Km3TZW5KJJ9XB2Mb0yIb3axUczzFwZMY0QDvnZtG1v2OdCM5lQ1g4UlPH+H8S/S/JURs/edqNeMz3m542QZ16SUL5uyj96hmZq/K9XT0r9t2SeE5fnrRtVsX2qMY8OpTXxjDzBrtvy1aHTZp6HJLYAv4Eum7NMKjSceu532PbCvWdfqE81f8/ztgG9erX19nepEM9M4skv2sTsD9/Fd1+ObY0A8f6FjVJqbr4bNsJLV/M3P+xqtzHaJ5q+N4rU50Mx2gmO1cp6/va8dI7ClHR31Ix/XZR/N81fHbrTDRGIYs7JzigK+iResZfs0tPs2VsuRfZQxrqeafxrwTbuZiXqQ6/nvb2Q7V7WvK+Bbd6R6jtWD5Np8I3rta9JlHHt8h0vzLzPPbpEeXzbgmyftZOU3dymHURzkJQyYkZV9rPY2LOOvBu6YXmOx7HOg5Qr4xobZE/BVhtned8Jj/PM024znH5rt0j1/vT2ugK99Fjv4yWxr/mZnYrY9ejU0fz3gS+k168FgG/sJLdo/e/6a1hnqjkmjHZWhSAOi9rEszb+elXGSgn4tNuYXDrTS1HkBV7W8UI/n6OmqKLXTeZykE8ouzxsR3G/E+E9jRlX2KXpScXn+mVRP7RDqt+OSfZTRTNMxzR9a6vmb+2Y8f3LvD+h59KnnrxdIS2WfVKe3PXVfCefkOupm59PuQPO3M5Wa2kAuvaTzxFgt9vwzlxhfR3aF2ls/pT7uwf4Ox2vp+TJxDuvYtudfszx//dhGnn+QbZNxHnJvM2aUd+BSef5OY+71/LPyXl47e0WM/zRmVD1/29O3UZ6/K8/fpfnPivV5Wz4CUiOuRm/aWn6q+Zv7duL5qx9vGvCFU/bxa/6ZQ2ZlnyAr+/g1f7dUkWj+RsA33T6Sffx5/q7O1c6w0dsxVguM71rJPj7PvEjzDwKzppP+ZEja/nmDrKLl6bpZ2v+Dq6RzYcDXcY5OA77i+QsdM6rlHYo6q8Tz1wyHneevyz6zxqJ/c5daYWeMqG2L8Gn+rvIOtuZvB3yTa2ilxr+e46kDxaNd88s7mPtm8vztgG+83Xg9QDvkJJZk4ypQRpah043jWN3U/FvteHBb4vrbxzI/j6Kg1wAAFGVJREFUq3vklH3aoZFJFHI6d7G6tV7PH2mev/7/kCnv4Az4msfK8/wzxt8X8K3ISovxn8Yk3nCBJz1sFBn/vQ2H5p8Y/7gome75j5nevI6efglkPX8f9kjMvFRPtUg3VvaMWOoa9Po+9v46djA0kX0Sz998gjIHebklJNWhGXn+huYf3RuXvOMjSAxtdBzdkGU1/xBjtSDZp6iYnSvg20qMv9n5tUPOdEQ+f5p0z1/733HV8+9uhK/7+oLAPam8eP5CxzTa01Tzn3Ro/nF8o9EKwcyGl59n/MetLJmePf+cbB89I8c1mUxLk2rysnMAf3kHZVzD0Jw8Rm9XxjvNyfPXz62kLpe840PvpOyyCX7N322cM3GO5H4iaas+F4Hezmg0brpdfEBvu53GX8/2iWWfovr9eSN8s9uar/byfiPGfxqTVrocLePvm1hdoTz/UCu5oGv+tlG19XkdWy7J6yh0xu08f/J7/kV5/opmK9WpzcFO2fN7vWAj4JveR71Z2Xr+6hip5p/IPgElZ+vK+GunCgJT1hhzpXrWtFTPAtlHT50FYtlHu6X7tNTTlmao01RLd5v1bB/9f8f+bl15/uoJJ9nHlQDgiWn4YgEyyEvoGFsKGRWarTDX21GaP5DN8tG9VkWu7FM3pQP1Y3cV2NLxBXzdP/boVRmPwAr4KhptTjpse5BX9phuL1hdRysu7KauQz+VL4VSr+2TBHw1Y6aedlyBcx96M2tkyhqZgG+LY9nHbRzLlHfQ76n+f6J76erVN45Az/YxPX9z+1bIzvpAejudAV+P5u/NAhLjL3SKbRhHhWY7xCHj/rJTeVk+zTZnKnr6cvgBrSyCpfkfPJH/BOAP+Pp/qHr9Hz0HXWFo/jltBpBx/e3rUCUjDornym21Q+/goozs09I1//RUqiPpWvaxslnG6w7Nvx744r3Zp5265flbt8zw/NucKS+d5/m7ZED7u3Vl+wCm7p+f7WMt93UKIvsInTLKmn+R8dW31V8bDs+/HpC3fINKkaxZP/aDC+Qff6pn9icVJoY01fxDds/d69L8XdhXM249wSjvXE2U3tIMlU+njmrjRO1QTdPnvlWebyeev2649BRKwK35jwWUXJz9xON92nFkEgGm8dc9/8JsH0oN84SR6ml+J612VvYBzJiKMwHAE/BNOgXx/IVeGdVsn0aB569je/4th+ZfC9zzyALZtDv1mD9rrJZbBsBn/F2djAq81rXMIl+2j0vzd+ENfsbtUN75QWPK+IeJ0bFtiWmQzRIOUXkHJWnFsk9Hmr/t+ZttbsWzhgGa5p/sax/M/DhuSV22ITZkHz3VMxnA5ZN93J6//Z20uYTn78z2KZB9Mpq/s5k9I8Z/GpPUNC+aGXvIaLbD0p5/o+WSfezSwH7jn2xjGf+xWpBrgDsp7KYbUkAv72Ab/7QeT1HMISOBWF7wAUv2aZfw/NV59acnvbzDmDp2l56/Ps2i3mb96W3MKmvhO5a+f1JCwtpeDQYEzIBvqvm7Ibg1f/u7ddXzj9qTLsub09mXJlomY6gfiPGfxuiyD7tGOA0pzXaIWfVaKa3TDmq7Ar55nn+yTXwu5emN1fP38ZV3cKV6Jp6/5tm1tXr+Oqo4WmFnVRD8VLn4yvNvahKFL+ALpAOvEqnKkH16zPaxAr5pLaE4NtWySjpnjpV/zbmefxhm8vzL1PbRR/ja323Lp/nrYypyPH9fRVAJ+Ao9o35UKid5VFBZH0UGEMhO2qGPTlUERBgvkFHsCU3GaoHT+1Zyjy37KEnH5em5PH81AbiN0qnHCgK+RXn+qsia4fkXBHyBWIdvsVP2Uca2M+Nvyz7muQBNsgvNks4Z42h7/tak9Xmaf6udyj7quHmaf5mArx5E1zEqmOYGfO0nMPNVIQFfoWOaRt2U0TH+KuujSPoATKOvPmc9/2JjanuPvs5HGVO7bXnlglU5hGy2T/Y7Ud5qkeZvY6esZj3/MFnny/OPzhtkB3kpzz9Qnn8HqZ7a+4BM2Ud50g3t6S2vw/fNYaBy6+177w/4Fnn+2ohvzwjf5JgFsk8nhd18AV/J8xc6xlXvfhRQIz2LDDaQrenjKjxWSvaxfpDjtcCYR1ehjOmENRK47vE+Ad3zT4OyoedpbK8y/gUFXXwBX1vzP1jz/L2yj3YspfknVT2D1OTWu/D8A8vzd8s+qgxzmDuy2b61+rb2GAIA2Dtp5/mn7XAdLzkP0k5qlj7Iy5Xn7ziGa+4C1/qyAd+qPP8qJ3AXBkzDkQ8/CqisjzLebzqKOZV/7Dz/TgK++mAtp+evBYTz9tdRBcaSx3qCM+ALAPsm2xkj6cJeW08MWuQFq6DsLE3zLxPwtT1/vaqnuuZOAr76qTIB37hzVd9h9MRHzn1dn40y1QFlDLHh+XNa2ycJ+HqzfbRKpobnb37nupSmUyvw/H0ylQR8hb7hqnc/CqjH/8KBTnBo/t6Ab/4PKJVEos9jtcDpfSvZx/6B1j1eG+D2/H0B372NVqlOL6P5100vWHnnyvi3Q32Ql7mvYfzrZGRM6VU97aeKMpijXa3gshXwjZ74AmN737H0/QE4J0LRPX9d81e7+e4ykV7V0y/7tMLQXbtH+7fJlX1sbd/TKYjxFzrGVfhsFGi0wg4Cvrbs48jzL+H52yN0fbKPMgb2mrxJwe1snzTg69L826WuO88QBgFlR/hqhc3yBk858/xVhxi3v5M8fx1Xnj+QThsZsuld2xq/fWeNDi/IToSy3y7sVjrbxxPwtbN92h7PvzDgm7bZtV8mz78iKy3Gfxoz0pp/nUoFfBt2wNcxwXhQwvO3g4FjHtlHHcc2oGkwN3vsVEJJjY8v4Lt3slXquvMkkMjzj0f4qkFebU6MaZHs02iZAd9U8+884GvXudGNoa75JyObc2Sf3A6PssbU8Pxdef4+zV87ljmZi7ldq0TAN2+9b75f8fyFnmm0Rl3z78Tz1/L8u/L8o9eibB/XZC3qHID7h9puZ/P8e/X8M15wzfSCDzhSPVXTigK+Rp5/kE4hmQR8HXP1ettpaf4+2Ud14vp1ZKeuNI+tyzCuOImZ7ROms4p55C+F3knpsk80yUu6U5naPnnrvXn+UxTw7cn4E9GbiGgFEYVEtMRa92EiWkVEDxPRK7Xl58fLVhHRh3o5v5DPqGv+pQK+luzTcBj/ICh+irDT73yppsoA2mY7N+DrG+HrUOL2NlpOuSmD/eRhecH2CN9WGCYdRt4k41G9HTaeVtTasSTVs/tsH3tMARAFfF3VTO1bmZGrAv+xgTRzCnB7/n7VXx/xrXdGpjH35vkb7coe2x/wde8zrJ7//QD+BMBN+kIiWgzgQgCnATgfwJeJqEZENQBfAnABgMUA3hJvK1TAqBp/NaNTqYBvi42JS5rtMCOn1ILiEbOp5x59Hvd0Pom3aZ3DZ1iByOsEzNo+Lc+4i32T3Xn+doaJneff0maysm2JM89fC/gmef5Jqmd32T72CF+VytvQqpkapawLjm1Xz8zk+WvlHUIt5tG150+EIEj3c03movbX98nIVx5HwdcpVGT7e0v1ZOYHAWfK1FIAP2DmSQBriGgVgLPidauY+dF4vx/E2z7QSzt87NjXwJu++vsqDj0SPLF9H4giO/XX3787yfkedtQwfzWxR15lis/+8mFcdvOjAKIfyYFmiIu/dyeAVFuvUfGYAWUk1VOCrvmr4xABh0zU42WeVE/HL1UdZ7wWn4MIu2M9Wj82M7BtXwNHzZ7Ibau6VvMcpvHfsbcBAJgdt1efHtEOpBoB33qAR7fsxT9e82ByLGUA1T3adaCZ2y5COn+AfqZ64Nb8P/KT+5KOvl5LO4iiwU268aw7Ar7b9jWS9xNjtUzA19fJjtUItZrD80ck+7UDRthm7G20nZ69Ph5grEaoB+m4Dv0e2E8uevvU/0V03mqsf1V5/icAuFX7vC5eBgBPWMtf6DoAEV0E4CIAWLBgQVeNCALComNmd7XvdGDRMbPxsmccjdvXbDMegYedZxx7KC549nE4c8ER+JMzT8CyNdtw8HgdTz96NmZP1PGrB5/E6fPmYP32/di0az8A4FnHHYZzn3k0rn/wSYTMOGvh0/AHpzwNv3l4M153xvHY12gjDBkXnH4sHtiwC08/ejaOPnRWcs53vuRk7J5s4R0vOgkHjdVw1klHYvOuSYzVArxi8TG44eHNOGP+HCw94wQcP+cgvHnJPGzf18Apc2dj/Y79OP2Ew/Gul56MJQuPwP9aehqeN/8I7J5sYtPOA3jJorlYdMxsvOiUpwEAXvvc4/Dk7gMIiJI2v2TRUbjz8R3YPdnEy049GgDww4vOxhPb9ydt/PyFZ+Bph0Qdw5+/aCFOmPMkzn/2sVi3fT8O1qqgvvucU3Dn2u04avYEzn3WMbjkvEV485L5+O2qrbhp5Vac96yjccrcQ9BsM+5dtwMvWHhksu//tWR+UgfqyEPGMf+Ig/B35z8Th86q40+fPw9PbN+HNVv34pS5s3HEweM4fs5BWLV5N8brAR7cuBt/9MyjcaDRxgMbd6HRDrHwqEOMe6zr9E8/ejbectZ87NwfdSZnzJ+DFz/9KHzgj5+BZWu24ZxT52b+N/7naxZj3hEHYdXmPZh7aNpJXvKKU3HSUYfgz85egIc27sajW/Zgw879WHzcYRirBTj3WUdj5ZN7cNhBY3j16ccBAF566lz8j5edgsXHH4b/um8Tzn/2sXhg4y6c96xjcKAV4l0vPRnPPuFw/PW5i7Bhx348d94c/N35z8ARh4zjlyueRDtkvOF58zJtfPc5p4AA3P3EDrztDxaiHgQ4fs5BWL1lDw4/aAynn3A4AOC4w2bhz1+0EM12iOefeETy/3jRS0/BZKuNW1ZuxdGHTZR6Au4GKir4RUTXAzjWseqjzPyf8TY3APgAMy+PP38RwK3M/O/x568D+K94v/OZ+Z3x8rcCeCEzvzevDUuWLOHly5eXvihBEAQBIKI7mHmJa12h58/M53VxzvUA5muf58XLkLNcEARBmCKqSvW8CsCFRDRBRCcBWARgGYDbASwiopOIaBxRUPiqitogCIIgeOhJ8yeiNwD4VwBzAVxNRHcz8yuZeQURXYEokNsCcDEzt+N93gvgWgA1AN9g5hU9XYEgCILQMYWa/zAgmr8gCELn5Gn+MsJXEARhBiLGXxAEYQYixl8QBGEGIsZfEARhBjISAV8i2gLg8R4OcRSArX1qTj+RdnWGtKszhrVdwPC2bbq160Rmzg6VxogY/14houW+iPcgkXZ1hrSrM4a1XcDwtm0mtUtkH0EQhBmIGH9BEIQZyEwx/pcNugEepF2dIe3qjGFtFzC8bZsx7ZoRmr8gCIJgMlM8f0EQBEFDjL8gCMIMZFob/2GaLJ6IHiOi+4jobiJSk94cSUTXEdHK+PWIKWrLN4hoMxHdry1ztoUivhDfw3uJ6MwpbtcniGh9fN/uJqJXaes+HLfrYSJ6ZYXtmk9EvyGiB4hoBRG9L14+0HuW066B3jMimkVEy4jonrhdn4yXn0REt8Xn/2Fc1h1x6fcfxstvI6KFU9yuy4lojXa/zoiXT9n/fny+GhHdRUQ/jz9Xe7+YeVr+ISoZvRrAyQDGAdwDYPEA2/MYgKOsZZ8B8KH4/YcA/H9T1JaXAjgTwP1FbQHwKkSzsBGAswHcNsXt+gSiWeLsbRfH3+kEgJPi77pWUbuOA3Bm/P5QAI/E5x/oPctp10DvWXzds+P3YwBui+/DFQAujJd/FcB74vf/A8BX4/cXAvhhRffL167LAbzRsf2U/e/H5/tbAN8D8PP4c6X3azp7/mchniyemRsA1GTxw8RSAN+K338LwOun4qTMfBOAbSXbshTAtzniVgBziOi4KWyXj6UAfsDMk8y8BsAqRN95Fe3ayMx3xu93A3gQ0ZzUA71nOe3yMSX3LL7uPfHHsfiPAbwcwI/i5fb9UvfxRwDOJSqYvb2/7fIxZf/7RDQPwKsB/J/4M6Hi+zWdjf8JyE4Wn/fDqBoG8EsiuoOiyekB4Bhm3hi/3wTgmME0Lbctw3Af3xs/dn9Dk8YG0q74Eft5iLzGoblnVruAAd+zWMK4G8BmANchesrYwcwtx7mTdsXrdwJ42lS0i5nV/bo0vl//TERqZvip/B7/BcAHAYTx56eh4vs1nY3/sPFiZj4TwAUALiail+orOXqGG4q822FqC4CvADgFwBkANgL43KAaQkSzAfwYwCXMvEtfN8h75mjXwO8ZM7eZ+QxE83SfBeCZU90GF3a7iOjZAD6MqH0vAHAkgL+byjYR0WsAbGbmO6byvNPZ+OdNIj/lMPP6+HUzgCsR/SCeVI+R8evmQbUvpy0DvY/M/GT8gw0BfA2pTDGl7SKiMUQG9rvM/JN48cDvmatdw3LP4rbsAPAbAH+ASDZRU8fq507aFa8/HMBTU9Su82P5jJl5EsA3MfX36w8BvI6IHkMkT78cwOdR8f2azsZ/aCaLJ6JDiOhQ9R7AHwO4P27P2+PN3g7gPwfRvhhfW64C8LY48+FsADs1qaNyLI31DYjum2rXhXHmw0kAFgFYVlEbCMDXATzIzP9bWzXQe+Zr16DvGRHNJaI58fuDALwCUTziNwDeGG9m3y91H98I4Nfxk9RUtOshrQMnRLq6fr8q/x6Z+cPMPI+ZFyKyU79m5j9D1fern9HqYftDFK1/BJHe+NEBtuNkRFkW9wBYodqCSKf7FYCVAK4HcOQUtef7iOSAJiIt8b/72oIo0+FL8T28D8CSKW7Xd+Lz3hv/0x+nbf/RuF0PA7igwna9GJGkcy+Au+O/Vw36nuW0a6D3DMBzANwVn/9+AB/TfgfLEAWa/wPARLx8Vvx5Vbz+5Clu16/j+3U/gH9HmhE0Zf/7WhtfhjTbp9L7JeUdBEEQZiDTWfYRBEEQPIjxFwRBmIGI8RcEQZiBiPEXBEGYgYjxFwRBmIGI8RcEQZiBiPEXBEGYgfz/hcuyGWPRrSQAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gyT7tNwkVdS-" + }, + "source": [ + "訓練時間\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "_t-JsKxUViFy", + "outputId": "333aa287-0455-4028-b91c-f83c8d2e1b57" + }, + "source": [ + "print(f\"total time is {end-start} sec\")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "total time is 674.2419369220734 sec\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "u2HaGRVEYGQS" + }, + "source": [ + "## 測試" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "5yFuUKKRYH73", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 500 + }, + "outputId": "7901d4d3-a71b-468e-a12e-6bd9edff551e" + }, + "source": [ + "fix(env, seed)\n", + "agent.network.eval() # 測試前先將 network 切換為 evaluation 模式\n", + "NUM_OF_TEST = 5 # Do not revise it !!!!!\n", + "test_total_reward = []\n", + "action_list = []\n", + "for i in range(NUM_OF_TEST):\n", + " actions = []\n", + " state = env.reset()\n", + "\n", + " img = plt.imshow(env.render(mode='rgb_array'))\n", + "\n", + " total_reward = 0\n", + "\n", + " done = False\n", + " while not done:\n", + " action, _ = agent.sample(state)\n", + " actions.append(action)\n", + " state, reward, done, _ = env.step(action)\n", + "\n", + " total_reward += reward\n", + "\n", + " #img.set_data(env.render(mode='rgb_array'))\n", + " #display.display(plt.gcf())\n", + " #display.clear_output(wait=True)\n", + " print(total_reward)\n", + " test_total_reward.append(total_reward)\n", + "\n", + " action_list.append(actions) #儲存你測試的結果\n", + " print(\"length of actions is \", len(actions))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/torch/__init__.py:422: UserWarning: torch.set_deterministic is deprecated and will be removed in a future release. Please use torch.use_deterministic_algorithms instead\n", + " \"torch.set_deterministic is deprecated and will be removed in a future \"\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "260.62265430635034\n", + "length of actions is 299\n", + "-212.89375915819693\n", + "length of actions is 319\n", + "11.862808485612831\n", + "length of actions is 241\n", + "8.015383611389638\n", + "length of actions is 231\n", + "-219.21903722619058\n", + "length of actions is 256\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAYXUlEQVR4nO3de5BV5Z3u8e9D0zS3RrobbNvuVkR7NFwcYBDxdgZRGSTWwakQhXNGjUNJrGgl1EzlROdUHZ06yVSlMmNmUpnCIeV1ktHxmIuU4wxj1FQm4y2oqAghoiDQIhi5yGVAuvt3/tir2y1N09fN7rf7+VTt2mu9a629fm+7fXrx9rv3UkRgZmbpGFLsAszMrHsc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiSlYcEuaL2mjpE2S7ijUeczMBhsVYh63pBLgt8BVwHbg18CSiFjf5yczMxtkCnXFPQvYFBHvRsQnwKPAwgKdy8xsUBlaoNetBbblrW8HLuxoZ0n++Kb1mZKSUsaUn8bwoWM50vQx+w/uIqKF8tHVDB86ttevf7hpL/sP7KS5+Sjl5dWMHFbF0eaDHDi0iyNHDvZBD8xyIkLHay9UcHdK0jJgWbHObwPX9OmLmH3eLQwrGc2adx/g+RceoLLyTC6/ZDnnjVuIdNz/F7okIvjN757guf/8W3btepv6+mlcNPXLVAyfwGtbf8ivnv8HDh/e34e9MWuvUEMljUB93npd1tYmIlZGxMyImFmgGmwQqqw8g/qaP2BMWT079r/O5i0v0tR0uEBnCzZvfpEd+9YydMgIaqumU1MzuUDnMvtUoYL710CDpLMkDQMWA6sKdC6zNhMnXkztKTM43LSPxt+9SmPjmwU936FDe3l3y/N8eGg9p46aytkTL2XYsJEFPadZQYZKIqJJ0u3AaqAEuD8i3irEucxaVVVNoP60mYwpq2Pz7l/wzjv/SUtLEwDNzUc59MluNu99ttfnOfTJbpqbj2Zrwdatr3DWGWsZV3sep1dOp67u93n33Rd6fR6zjhRsjDsingKeKtTrm+WThjDhzFmcNmYKh5s+pvGj13j//XVt2z/6aDOvvPYIZWXlvT7XkSP7+eijzW3rBw9+xJZtL3J6xXROG30+EydexLZtr3H0aKGGaGywK9ofJ836Wks08+HBjUSsZ9Om/yCi5TPb33+/cP/o27btNd6ve5WoaGbfvg8YPXo8e/Zs6/xAsx4oyAdwul2EpwNaHygpKaWi4gxaWprYu7exbZjkZBk7tpaamsn89re/oLn5k5N6bhuYOpoO6OA2M+unOgpuf8mUmVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJaZXd8CRtAXYDzQDTRExU1Il8M/ABGALcF1E7OldmWZm1qovrrgvj4hpETEzW78DeCYiGoBnsnUzM+sjhRgqWQg8lC0/BFxbgHOYmQ1avQ3uAP5d0iuSlmVt1RGxI1v+AKju5TnMzCxPb+/yfmlENEo6FXha0m/yN0ZEdHQ/ySzolx1vm5mZdazPbhYs6W7gAHALMCcidkiqAX4REed2cqxvFmxmdow+v1mwpFGSyluXgXnAOmAVcFO2203AEz09h5mZtdfjK25JE4GfZqtDgX+KiG9JqgIeA84A3iM3HXB3J6/lK24zs2N0dMXdZ0MlveHgNjNrr8+HSszMrDgc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klptPglnS/pF2S1uW1VUp6WtLb2XNF1i5J35O0SdIbkmYUsngzs8GoK1fcDwLzj2m7A3gmIhqAZ7J1gKuBhuyxDFjRN2WamVmrToM7In4J7D6meSHwULb8EHBtXvvDkfMiMFZSTV8Va2ZmPR/jro6IHdnyB0B1tlwLbMvbb3vW1o6kZZLWSFrTwxrMzAalob19gYgISdGD41YCKwF6cryZ2WDV0yvuna1DINnzrqy9EajP268uazMzsz7S0+BeBdyULd8EPJHXfmM2u2Q2sC9vSMXMzPqAIk48SiHpEWAOMA7YCdwF/Ax4DDgDeA+4LiJ2SxLwfXKzUA4BN0dEp2PYHioxM2svInS89k6D+2RwcJuZtddRcPuTk2ZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klptPglnS/pF2S1uW13S2pUdLa7LEgb9udkjZJ2ijpjwpVuJnZYNWVmwX/N+AA8HBETMna7gYORMRfH7PvJOARYBZwOvBz4PciormTc/iek2Zmx+jxPScj4pfA7i6eZyHwaEQciYjNwCZyIW5mZn2kN2Pct0t6IxtKqcjaaoFteftsz9rakbRM0hpJa3pRg5nZoNPT4F4BnA1MA3YAf9PdF4iIlRExMyJm9rAGM7NBqUfBHRE7I6I5IlqAH/DpcEgjUJ+3a13WZmZmfaRHwS2pJm/1j4HWGSergMWSyiSdBTQAL/euRDMzyze0sx0kPQLMAcZJ2g7cBcyRNA0IYAvwZYCIeEvSY8B6oAm4rbMZJWZm1j2dTgc8KUV4OqCZWTs9ng5oZmb9i4PbzCwxDm4zs8Q4uM3MEuPgNjNLjIPbzCwxDm4zs8Q4uM3MEuPgNjNLjIPbzCwxDm4zs8Q4uM3MEuPgNjNLjIPbzCwxDm4zs8Q4uM3MEuPgNjNLjIPbzCwxnQa3pHpJz0laL+ktSV/L2islPS3p7ey5ImuXpO9J2iTpDUkzCt0JM7PBpCtX3E3An0fEJGA2cJukScAdwDMR0QA8k60DXE3u7u4NwDJgRZ9XbWY2iHUa3BGxIyJezZb3AxuAWmAh8FC220PAtdnyQuDhyHkRGCupps8rNzMbpLo1xi1pAjAdeAmojogd2aYPgOpsuRbYlnfY9qzt2NdaJmmNpDXdrNnMbFDrcnBLGg38GFgeER/nb4uIAKI7J46IlRExMyJmduc4M7PBrkvBLamUXGj/KCJ+kjXvbB0CyZ53Ze2NQH3e4XVZm5mZ9YGuzCoRcB+wISLuydu0CrgpW74JeCKv/cZsdslsYF/ekIqZmfWScqMcJ9hBuhT4D+BNoCVr/gty49yPAWcA7wHXRcTuLOi/D8wHDgE3R8QJx7EldWuYxcxsMIgIHa+90+A+GRzcZmbtdRTc/uSkmVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZonpys2C6yU9J2m9pLckfS1rv1tSo6S12WNB3jF3StokaaOkPypkB8zMBpuu3Cy4BqiJiFcllQOvANcC1wEHIuKvj9l/EvAIMAs4Hfg58HsR0XyCc/iek2Zmx+jxPScjYkdEvJot7wc2ALUnOGQh8GhEHImIzcAmciFuZmZ9oFtj3JImANOBl7Km2yW9Iel+SRVZWy2wLe+w7Zw46M0A+Ku/+jLf/jZMmQKTJsHppxe7opNvzpw5PPjguSxYAJMnw3nnQUlJsauy/mZoV3eUNBr4MbA8Ij6WtAL4v0Bkz38D/Gk3Xm8ZsKx75dpANnXqRGpqYO7c3PqOHbB+fW753/4NNm2CCPjgA2jucOAtbePHj2fWrANMnpxbb2qC55+Ho0dh+3b42c9y7fv2wf79xavTiqtLwS2plFxo/ygifgIQETvztv8AeDJbbQTq8w6vy9o+IyJWAiuz4z3GbW2UjeqdfvqnV92XX54L7eZmWL0a/uu/csH+wx8Wr85Cav0ZlJbCH/5hbjkC/uRPcsvr1sHGjbnlhx+GnTvbv4YNXF2ZVSLgPmBDRNyT116Tt9sfA+uy5VXAYkllks4CGoCX+65kG4xaWnKh3dQEhw7BwYO58B5MWn9xNTfD4cO5n8HBg7mfjQ0uXbnivgS4AXhT0tqs7S+AJZKmkRsq2QJ8GSAi3pL0GLAeaAJuO9GMErN8EbkH5IYG1mbvuNWr4d13c9t27x74YdX6c2hqgmefhU8+gcZGWLUqt/3AgcH3i8s+1WlwR8SvgONNSXnqBMd8C/hWL+qyQejAAfiXf8kNf7S05MZwP/yw2FWdfGvXwg9+AO+9l/s5bN068H9RWfd0+Y+TZoW2dSvcfXexqyi+e+6BNWuKXYX1Zw5uM7M8o0ePpqamhvnz57N69WqO/ZDikSNH2Lp1a5Gqy3Fwm9mgV15ezpgxY/jqV7/KlClTmDdvHkOGDKHlOGNUu3fv5vHHH28X6Dt37uTee+9tt39TUxN79uzp03od3GY26JSUlDBs2DBqa2u5/vrrufrqqzn//PMZNWoUQ4Z8Otkuf7nVqaeeyle+8pV27c3NzXz9619v175r1y4eeOCB47b/8DjzWVtaWjhy5MgJ63dwm9mgIImRI0fyhS98gYsvvphrrrmGsrIyxo0b1yevX1JSQnl5ebv28vJyvvnNb7ZrP3r0KHfddVe79h07drBixQqeeOKJDs/l4DazAa26upqZM2fypS99ialTp9LQ0HDcK+mTrbS0lNra9t8GUltby3333cfrr7/e4bEObjMbcCZOnEhdXR3f+MY3mDBhApMmTSp2SX3KwW1mA8KECRNoaGjg1ltvZcaMGZx55plIx/1W1OQ5uM0sSSUlJVRVVTFnzhwuv/xyFi1aREVFBSWD4OsUHdxmlozW2SCLFy9m6tSpLF26lLKyMsrKyopd2knl4Dazfk0SI0aMYNGiRcyePZuFCxcybtw4hg0bVuzSisbBbWb9UnV1NRdccAE33ngj559/fr+ZDdIfOLjNrOgmT57MmDFjALjiiiuYN28elZWVTG69o4R9hoPbzE6a+vp6RowYAcDcuXOZN28eAJdddhlVVVVt+w3U2SB9xcFtZn2uoqKibQz6sssu44orrgDgmmuu4bTTTgNyHyf30EfPOLjNrMfKysrapt9deOGFXHbZZQAsXryY+vrcHQyHDRs2qP+QWAgObjPrktar46lTp3LBBRcAcPPNN3PWWWcBMGrUqLZxaissB7eZHdfZZ5/d9lHxuro6br/9dgCqqqqorq4uZmmDXqfBLWk48EugLNv/8Yi4K7sR8KNAFfAKcENEfCKpDHgY+APgI+D6iNhSoPrNrIcaGho+88145557LkuXLm1br62tbbuatv6lK1fcR4C5EXFAUinwK0n/CvwZ8N2IeFTSvcBSYEX2vCcizpG0GPg2cH2B6jezY1RUVHDqqad+pu3zn/982/hzq5kzZ7b7djrP5khDV24WHMCBbLU0ewQwF/gfWftDwN3kgnthtgzwOPB9SYpjbxdhZt2WP1uj1a233kpNTU3b+pQpU5g9e/Zn9pHkGRwDSJfGuCWVkBsOOQf4e+AdYG9ENGW7bAdaf3XXAtsAIqJJ0j5ywym/68O6LVElJSUdfq/E888/z6hRo9rdEmow2bJlC/v27WPkyJEsWLCg3QdQFi1a1G74Yvjw4YPii5XsU10K7ohoBqZJGgv8FDivtyeWtAxY1tvXsd6R1KN/Hk+dOpULL7yw28edc8453HDDDd0+bjA65ZRT2j6sYpavW7NKImKvpOeAi4CxkoZmV911QGO2WyNQD2yXNBQ4hdwfKY99rZXASgBJg/cSq49Nnz6dM844o8v7X3fddcyYMaPb56msrGw3jmpmJ0dXZpWMB45moT0CuIrcHxyfAxaRm1lyE9B6g7RV2foL2fZnPb7dsTFjxjB16tQOt3/xi19smzPbFeecc44D1WyA68oVdw3wUDbOPQR4LCKelLQeeFTSN4HXgPuy/e8D/lHSJmA3sLgAdSfnzDPPZOTIkQBcddVVbR8Brqys5NJLLy1maWaWmK7MKnkDmH6c9neBWcdpPwx8sU+qS1BVVRWlpaWUl5ezfPlySktLgdx3NLReCfsv/GbWG/3ik5OSkppJUFJSwvDhw4Hcx3xvueWWtoBesmQJtbW1bV/+7nmxZtbX+kVwT5w4kSuvvPKE+zz11FO8//77HW6PCFpaWvq0riFDhrQF7yWXXMLnPvc5IPcJsyVLlgCf3vfOV9BmdrL0i+AeO3Ys99577wn32bZtG4cOHepw+8GDB/nOd77D4cOHO9xn586dvPDCCx1uHzVqFFdeeWXbFLnly5e3fSdDdXU1Y8eO7aQnZmaF1y+CuytavyLyRB555JETbt+zZw8bNmzocPuIESOYNm2ahzfMrF9LJrj7QkVFBRdffHGxyzAz6xUPzJqZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJabT4JY0XNLLkl6X9Jakv8zaH5S0WdLa7DEta5ek70naJOkNSd2/hbiZmXWoK1/regSYGxEHJJUCv5L0r9m2r0fE48fsfzXQkD0uBFZkz2Zm1gc6veKOnAPZamn2ONENIhcCD2fHvQiMlVTT+1LNzAy6OMYtqUTSWmAX8HREvJRt+lY2HPJdSWVZWy2wLe/w7VmbmZn1gS4Fd0Q0R8Q0oA6YJWkKcCdwHnABUAl8ozsnlrRM0hpJaz788MNulm1mNnh1a1ZJROwFngPmR8SObDjkCPAAMCvbrRHIv0FkXdZ27GutjIiZETFz/PjxPavezGwQ6sqskvGSxmbLI4CrgN+0jlsrd2fda4F12SGrgBuz2SWzgX0RsaMg1ZuZDUJdmVVSAzwkqYRc0D8WEU9KelbSeEDAWuDWbP+ngAXAJuAQcHPfl21mNnh1GtwR8QYw/TjtczvYP4Dbel+amZkdjz85aWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliFBHFrgFJ+4GNxa6jQMYBvyt2EQUwUPsFA7dv7ldazoyI8cfbMPRkV9KBjRExs9hFFIKkNQOxbwO1XzBw++Z+DRweKjEzS4yD28wsMf0luFcWu4ACGqh9G6j9goHbN/drgOgXf5w0M7Ou6y9X3GZm1kVFD25J8yVtlLRJ0h3Frqe7JN0vaZekdXltlZKelvR29lyRtUvS97K+viFpRvEqPzFJ9ZKek7Re0luSvpa1J903ScMlvSzp9axff5m1nyXppaz+f5Y0LGsvy9Y3ZdsnFLP+zkgqkfSapCez9YHSry2S3pS0VtKarC3p92JvFDW4JZUAfw9cDUwClkiaVMyaeuBBYP4xbXcAz0REA/BMtg65fjZkj2XAipNUY080AX8eEZOA2cBt2X+b1Pt2BJgbEb8PTAPmS5oNfBv4bkScA+wBlmb7LwX2ZO3fzfbrz74GbMhbHyj9Arg8IqblTf1L/b3YcxFRtAdwEbA6b/1O4M5i1tTDfkwA1uWtbwRqsuUacvPUAf4BWHK8/fr7A3gCuGog9Q0YCbwKXEjuAxxDs/a29yWwGrgoWx6a7adi195Bf+rIBdhc4ElAA6FfWY1bgHHHtA2Y92J3H8UeKqkFtuWtb8/aUlcdETuy5Q+A6mw5yf5m/4yeDrzEAOhbNpywFtgFPA28A+yNiKZsl/za2/qVbd8HVJ3cirvsb4H/BbRk61UMjH4BBPDvkl6RtCxrS/692FP95ZOTA1ZEhKRkp+5IGg38GFgeER9LatuWat8iohmYJmks8FPgvCKX1GuSrgF2RcQrkuYUu54CuDQiGiWdCjwt6Tf5G1N9L/ZUsa+4G4H6vPW6rC11OyXVAGTPu7L2pPorqZRcaP8oIn6SNQ+IvgFExF7gOXJDCGMltV7I5Nfe1q9s+ynARye51K64BPjvkrYAj5IbLvk70u8XABHRmD3vIvfLdhYD6L3YXcUO7l8DDdlfvocBi4FVRa6pL6wCbsqWbyI3PtzafmP2V+/ZwL68f+r1K8pdWt8HbIiIe/I2Jd03SeOzK20kjSA3br+BXIAvynY7tl+t/V0EPBvZwGl/EhF3RkRdREwg9//RsxHxP0m8XwCSRkkqb10G5gHrSPy92CvFHmQHFgC/JTfO+L+LXU8P6n8E2AEcJTeWtpTcWOEzwNvAz4HKbF+Rm0XzDvAmMLPY9Z+gX5eSG1d8A1ibPRak3jfgfOC1rF/rgP+TtU8EXgY2Af8PKMvah2frm7LtE4vdhy70cQ7w5EDpV9aH17PHW605kfp7sTcPf3LSzCwxxR4qMTOzbnJwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWL+P8eEF+CNkxTNAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "Aex7mcKr0J01", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "f1706a79-1fbd-4d61-bdcd-ab257cb152e5" + }, + "source": [ + "print(f\"Your final reward is : %.2f\"%np.mean(test_total_reward))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Your final reward is : -30.32\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "leyebGYRpqsF" + }, + "source": [ + "Action list 的長相" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "hGAH4YWDpp4u", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "c7f5fa21-7b7a-43a8-8478-df76dce7a4ad" + }, + "source": [ + "print(\"Action list looks like \", action_list)\n", + "print(\"Action list's shape looks like \", np.shape(action_list))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Action list looks like [[1, 1, 1, 1, 2, 2, 1, 2, 1, 2, 1, 2, 2, 2, 2, 1, 2, 2, 1, 2, 1, 2, 1, 2, 2, 2, 1, 1, 2, 2, 1, 2, 2, 1, 2, 3, 2, 2, 3, 2, 2, 3, 3, 3, 2, 2, 3, 2, 3, 3, 3, 3, 3, 3, 2, 2, 3, 3, 3, 3, 3, 2, 3, 2, 2, 3, 3, 3, 2, 2, 3, 2, 2, 3, 2, 3, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 1, 1, 2, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 2, 1, 1, 2, 1, 1, 1, 2, 1, 1, 1, 2, 2, 1, 1, 2, 1, 2, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 1, 2, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 2, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 3, 2, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 2, 3, 2, 2, 2, 2, 3, 2, 2, 3, 2, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 2, 2, 2, 2, 2, 2, 3, 2, 2, 3, 2, 0, 2, 2, 3, 2, 0, 3, 2, 2, 2, 3, 2, 2, 3, 2, 3, 2, 3, 2, 2, 2, 3, 2, 3, 2, 2, 0, 1, 2, 2, 2, 0, 1, 2, 2, 1, 2, 1, 2, 1, 1, 2, 2, 0, 2, 1, 2, 2, 1, 1, 1, 2, 2, 2, 1, 2, 2, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2, 1, 2, 2, 0, 1, 0, 2, 2, 2, 2, 3, 3, 2, 3, 2, 3, 2, 3, 3, 2, 3, 2, 3, 3, 3, 2, 3, 2, 3, 3, 3, 2, 2, 3, 3, 3, 2, 2, 3, 2, 3, 2, 3, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 1, 1, 1, 2, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 2, 1, 1, 1, 2, 2, 1, 2, 1, 1, 2, 1, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 3, 2, 2, 0, 3, 3, 3, 3, 3, 2, 3, 3, 3, 3, 3, 2, 3, 2, 3, 3, 3, 3, 2, 3, 3, 3, 3, 2, 2, 2, 3, 3, 3, 3, 3, 2, 2, 3, 3, 2, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 2], [2, 1, 2, 2, 1, 1, 2, 2, 1, 1, 1, 2, 2, 2, 2, 2, 1, 2, 2, 2, 0, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 3, 3, 3, 3, 2, 3, 3, 2, 2, 2, 3, 3, 3, 3, 2, 2, 3, 3, 3, 3, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 2, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 2, 2, 1, 1, 2, 1, 2, 2, 1, 2, 2, 1, 1, 2, 2, 2, 2, 1, 2, 1, 2, 1, 2, 2, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 3, 3, 3, 3, 3, 3, 2, 3, 2, 3, 3, 3, 3, 3, 3, 3, 2, 2, 3, 2, 3, 3, 3, 3, 3, 2, 3, 3, 3, 2, 2, 2, 2, 3, 3, 3, 2, 2, 3, 3, 2, 3, 3, 2, 2, 2, 2, 2], [1, 1, 1, 1, 2, 1, 2, 2, 1, 1, 2, 2, 1, 2, 2, 1, 2, 1, 2, 2, 2, 1, 2, 1, 2, 2, 2, 1, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 3, 0, 2, 2, 3, 3, 2, 3, 3, 3, 3, 2, 3, 2, 3, 3, 3, 2, 3, 3, 2, 3, 3, 2, 3, 2, 2, 3, 3, 3, 2, 3, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 1, 2, 1, 1, 1, 2, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 2, 1, 1, 2, 1, 1, 2, 2, 1, 1, 1, 2, 1, 1, 2, 2, 1, 2, 1, 1, 2, 1, 2, 2, 2, 1, 1, 2, 2, 1, 2, 2, 2, 2, 1, 1, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 2, 3, 3, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 3, 3, 2, 3, 3, 3, 2, 3, 2, 2, 3, 3, 3, 2, 3, 2, 3, 2, 3, 2, 2, 2, 2, 3, 2, 2, 2, 2], [1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 2, 2, 0, 1, 2, 2, 0, 2, 3, 2, 3, 2, 3, 2, 3, 2, 2, 3, 2, 3, 3, 2, 3, 3, 2, 2, 3, 3, 3, 2, 2, 3, 2, 3, 2, 3, 3, 2, 3, 2, 2, 2, 3, 2, 3, 2, 2, 2, 2, 3, 2, 3, 2, 2, 2, 2, 3, 2, 2, 1, 1, 2, 1, 1, 2, 1, 1, 2, 2, 1, 2, 2, 1, 2, 2, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1, 2, 1, 2, 2, 1, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 2, 1, 2, 2, 2, 2, 2, 3, 3, 2, 3, 3, 2, 3, 3, 3, 2, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 3, 3, 3, 2, 2, 3, 3, 3, 2, 3, 2, 3, 3, 2, 2, 3, 3, 2, 3, 2, 2, 2, 3, 2, 2, 3, 2, 2, 2, 3, 2, 2, 2, 2, 2, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]\n", + "Action list's shape looks like (5,)\n" + ], + "name": "stdout" + }, + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n", + " return array(a, dtype, copy=False, order=order)\n" + ], + "name": "stderr" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "l7sokqEUtrFY" + }, + "source": [ + "Action 的分布\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "WHdAItjj1nxw", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "5129773b-1f4a-4085-d2bf-3bc2abc2598c" + }, + "source": [ + "distribution = {}\n", + "for actions in action_list:\n", + " for action in actions:\n", + " if action not in distribution.keys():\n", + " distribution[action] = 1\n", + " else:\n", + " distribution[action] += 1\n", + "print(distribution)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "{1: 278, 2: 698, 3: 297, 0: 73}\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ricE0schY75M" + }, + "source": [ + "儲存 Model Testing的結果\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "GZsMkGmIY42b", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "8c55c932-4654-4f8c-f6b0-fa52ac3e8b96" + }, + "source": [ + "PATH = \"Action_List_test.npy\" # 可以改成你想取的名字或路徑\n", + "np.save(PATH ,np.array(action_list)) " + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:2: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n", + " \n" + ], + "name": "stderr" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "asK7WfbkaLjt" + }, + "source": [ + "### 你要交到JudgeBoi的檔案94這個\n", + "儲存結果到本地端 (就是你的電腦裡拉 = = )\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "c-CqyhHzaWAL", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 17 + }, + "outputId": "adfba5e6-a107-49aa-9f98-3c0655c5d6c2" + }, + "source": [ + "from google.colab import files\n", + "files.download(PATH)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "\n", + " async function download(id, filename, size) {\n", + " if (!google.colab.kernel.accessAllowed) {\n", + " return;\n", + " }\n", + " const div = document.createElement('div');\n", + " const label = document.createElement('label');\n", + " label.textContent = `Downloading \"${filename}\": `;\n", + " div.appendChild(label);\n", + " const progress = document.createElement('progress');\n", + " progress.max = size;\n", + " div.appendChild(progress);\n", + " document.body.appendChild(div);\n", + "\n", + " const buffers = [];\n", + " let downloaded = 0;\n", + "\n", + " const channel = await google.colab.kernel.comms.open(id);\n", + " // Send a message to notify the kernel that we're ready.\n", + " channel.send({})\n", + "\n", + " for await (const message of channel.messages) {\n", + " // Send a message to notify the kernel that we're ready.\n", + " channel.send({})\n", + " if (message.buffers) {\n", + " for (const buffer of message.buffers) {\n", + " buffers.push(buffer);\n", + " downloaded += buffer.byteLength;\n", + " progress.value = downloaded;\n", + " }\n", + " }\n", + " }\n", + " const blob = new Blob(buffers, {type: 'application/binary'});\n", + " const a = document.createElement('a');\n", + " a.href = window.URL.createObjectURL(blob);\n", + " a.download = filename;\n", + " div.appendChild(a);\n", + " a.click();\n", + " div.remove();\n", + " }\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + } + }, + { + "output_type": "display_data", + "data": { + "application/javascript": [ + "download(\"download_5d13b99b-295d-4ab0-814c-b2d0fff26eff\", \"Action_List_test.npy\", 2999)" + ], + "text/plain": [ + "" + ] + }, + "metadata": { + "tags": [] + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "seT4NUmWmAZ1" + }, + "source": [ + "# Server 測試\n", + "到時候下面會是我們Server上測試的環境，可以給大家看一下自己的表現如何" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "U69c-YTxaw6b", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 412 + }, + "outputId": "50015892-29ae-4665-c66f-880aecf7be8f" + }, + "source": [ + "action_list = np.load(PATH,allow_pickle=True) #到時候你上傳的檔案\n", + "seed = 543 #到時候測試的seed 請不要更改\n", + "fix(env, seed)\n", + "\n", + "agent.network.eval() # 測試前先將 network 切換為 evaluation 模式\n", + "\n", + "test_total_reward = []\n", + "for actions in action_list:\n", + " state = env.reset()\n", + " img = plt.imshow(env.render(mode='rgb_array'))\n", + "\n", + " total_reward = 0\n", + "\n", + " done = False\n", + " # while not done:\n", + " done_count = 0\n", + " for action in actions:\n", + " # action, _ = agent1.sample(state)\n", + " state, reward, done, _ = env.step(action)\n", + " done_count += 1\n", + " total_reward += reward\n", + " if done:\n", + " \n", + " break\n", + " # img.set_data(env.render(mode='rgb_array'))\n", + " # display.display(plt.gcf())\n", + " # display.clear_output(wait=True)\n", + " print(f\"Your reward is : %.2f\"%total_reward)\n", + " test_total_reward.append(total_reward)" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.7/dist-packages/torch/__init__.py:422: UserWarning: torch.set_deterministic is deprecated and will be removed in a future release. Please use torch.use_deterministic_algorithms instead\n", + " \"torch.set_deterministic is deprecated and will be removed in a future \"\n" + ], + "name": "stderr" + }, + { + "output_type": "stream", + "text": [ + "Your reward is : 260.62\n", + "Your reward is : -212.89\n", + "Your reward is : 11.86\n", + "Your reward is : 8.02\n", + "Your reward is : -219.22\n" + ], + "name": "stdout" + }, + { + "output_type": "display_data", + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAYXUlEQVR4nO3de5BV5Z3u8e9D0zS3RrobbNvuVkR7NFwcYBDxdgZRGSTWwakQhXNGjUNJrGgl1EzlROdUHZ06yVSlMmNmUpnCIeV1ktHxmIuU4wxj1FQm4y2oqAghoiDQIhi5yGVAuvt3/tir2y1N09fN7rf7+VTt2mu9a629fm+7fXrx9rv3UkRgZmbpGFLsAszMrHsc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiSlYcEuaL2mjpE2S7ijUeczMBhsVYh63pBLgt8BVwHbg18CSiFjf5yczMxtkCnXFPQvYFBHvRsQnwKPAwgKdy8xsUBlaoNetBbblrW8HLuxoZ0n++Kb1mZKSUsaUn8bwoWM50vQx+w/uIqKF8tHVDB86ttevf7hpL/sP7KS5+Sjl5dWMHFbF0eaDHDi0iyNHDvZBD8xyIkLHay9UcHdK0jJgWbHObwPX9OmLmH3eLQwrGc2adx/g+RceoLLyTC6/ZDnnjVuIdNz/F7okIvjN757guf/8W3btepv6+mlcNPXLVAyfwGtbf8ivnv8HDh/e34e9MWuvUEMljUB93npd1tYmIlZGxMyImFmgGmwQqqw8g/qaP2BMWT079r/O5i0v0tR0uEBnCzZvfpEd+9YydMgIaqumU1MzuUDnMvtUoYL710CDpLMkDQMWA6sKdC6zNhMnXkztKTM43LSPxt+9SmPjmwU936FDe3l3y/N8eGg9p46aytkTL2XYsJEFPadZQYZKIqJJ0u3AaqAEuD8i3irEucxaVVVNoP60mYwpq2Pz7l/wzjv/SUtLEwDNzUc59MluNu99ttfnOfTJbpqbj2Zrwdatr3DWGWsZV3sep1dOp67u93n33Rd6fR6zjhRsjDsingKeKtTrm+WThjDhzFmcNmYKh5s+pvGj13j//XVt2z/6aDOvvPYIZWXlvT7XkSP7+eijzW3rBw9+xJZtL3J6xXROG30+EydexLZtr3H0aKGGaGywK9ofJ836Wks08+HBjUSsZ9Om/yCi5TPb33+/cP/o27btNd6ve5WoaGbfvg8YPXo8e/Zs6/xAsx4oyAdwul2EpwNaHygpKaWi4gxaWprYu7exbZjkZBk7tpaamsn89re/oLn5k5N6bhuYOpoO6OA2M+unOgpuf8mUmVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJaZXd8CRtAXYDzQDTRExU1Il8M/ABGALcF1E7OldmWZm1qovrrgvj4hpETEzW78DeCYiGoBnsnUzM+sjhRgqWQg8lC0/BFxbgHOYmQ1avQ3uAP5d0iuSlmVt1RGxI1v+AKju5TnMzCxPb+/yfmlENEo6FXha0m/yN0ZEdHQ/ySzolx1vm5mZdazPbhYs6W7gAHALMCcidkiqAX4REed2cqxvFmxmdow+v1mwpFGSyluXgXnAOmAVcFO2203AEz09h5mZtdfjK25JE4GfZqtDgX+KiG9JqgIeA84A3iM3HXB3J6/lK24zs2N0dMXdZ0MlveHgNjNrr8+HSszMrDgc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klptPglnS/pF2S1uW1VUp6WtLb2XNF1i5J35O0SdIbkmYUsngzs8GoK1fcDwLzj2m7A3gmIhqAZ7J1gKuBhuyxDFjRN2WamVmrToM7In4J7D6meSHwULb8EHBtXvvDkfMiMFZSTV8Va2ZmPR/jro6IHdnyB0B1tlwLbMvbb3vW1o6kZZLWSFrTwxrMzAalob19gYgISdGD41YCKwF6cryZ2WDV0yvuna1DINnzrqy9EajP268uazMzsz7S0+BeBdyULd8EPJHXfmM2u2Q2sC9vSMXMzPqAIk48SiHpEWAOMA7YCdwF/Ax4DDgDeA+4LiJ2SxLwfXKzUA4BN0dEp2PYHioxM2svInS89k6D+2RwcJuZtddRcPuTk2ZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klptPglnS/pF2S1uW13S2pUdLa7LEgb9udkjZJ2ijpjwpVuJnZYNWVmwX/N+AA8HBETMna7gYORMRfH7PvJOARYBZwOvBz4PciormTc/iek2Zmx+jxPScj4pfA7i6eZyHwaEQciYjNwCZyIW5mZn2kN2Pct0t6IxtKqcjaaoFteftsz9rakbRM0hpJa3pRg5nZoNPT4F4BnA1MA3YAf9PdF4iIlRExMyJm9rAGM7NBqUfBHRE7I6I5IlqAH/DpcEgjUJ+3a13WZmZmfaRHwS2pJm/1j4HWGSergMWSyiSdBTQAL/euRDMzyze0sx0kPQLMAcZJ2g7cBcyRNA0IYAvwZYCIeEvSY8B6oAm4rbMZJWZm1j2dTgc8KUV4OqCZWTs9ng5oZmb9i4PbzCwxDm4zs8Q4uM3MEuPgNjNLjIPbzCwxDm4zs8Q4uM3MEuPgNjNLjIPbzCwxDm4zs8Q4uM3MEuPgNjNLjIPbzCwxDm4zs8Q4uM3MEuPgNjNLjIPbzCwxnQa3pHpJz0laL+ktSV/L2islPS3p7ey5ImuXpO9J2iTpDUkzCt0JM7PBpCtX3E3An0fEJGA2cJukScAdwDMR0QA8k60DXE3u7u4NwDJgRZ9XbWY2iHUa3BGxIyJezZb3AxuAWmAh8FC220PAtdnyQuDhyHkRGCupps8rNzMbpLo1xi1pAjAdeAmojogd2aYPgOpsuRbYlnfY9qzt2NdaJmmNpDXdrNnMbFDrcnBLGg38GFgeER/nb4uIAKI7J46IlRExMyJmduc4M7PBrkvBLamUXGj/KCJ+kjXvbB0CyZ53Ze2NQH3e4XVZm5mZ9YGuzCoRcB+wISLuydu0CrgpW74JeCKv/cZsdslsYF/ekIqZmfWScqMcJ9hBuhT4D+BNoCVr/gty49yPAWcA7wHXRcTuLOi/D8wHDgE3R8QJx7EldWuYxcxsMIgIHa+90+A+GRzcZmbtdRTc/uSkmVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZonpys2C6yU9J2m9pLckfS1rv1tSo6S12WNB3jF3StokaaOkPypkB8zMBpuu3Cy4BqiJiFcllQOvANcC1wEHIuKvj9l/EvAIMAs4Hfg58HsR0XyCc/iek2Zmx+jxPScjYkdEvJot7wc2ALUnOGQh8GhEHImIzcAmciFuZmZ9oFtj3JImANOBl7Km2yW9Iel+SRVZWy2wLe+w7Zw46M0A+Ku/+jLf/jZMmQKTJsHppxe7opNvzpw5PPjguSxYAJMnw3nnQUlJsauy/mZoV3eUNBr4MbA8Ij6WtAL4v0Bkz38D/Gk3Xm8ZsKx75dpANnXqRGpqYO7c3PqOHbB+fW753/4NNm2CCPjgA2jucOAtbePHj2fWrANMnpxbb2qC55+Ho0dh+3b42c9y7fv2wf79xavTiqtLwS2plFxo/ygifgIQETvztv8AeDJbbQTq8w6vy9o+IyJWAiuz4z3GbW2UjeqdfvqnV92XX54L7eZmWL0a/uu/csH+wx8Wr85Cav0ZlJbCH/5hbjkC/uRPcsvr1sHGjbnlhx+GnTvbv4YNXF2ZVSLgPmBDRNyT116Tt9sfA+uy5VXAYkllks4CGoCX+65kG4xaWnKh3dQEhw7BwYO58B5MWn9xNTfD4cO5n8HBg7mfjQ0uXbnivgS4AXhT0tqs7S+AJZKmkRsq2QJ8GSAi3pL0GLAeaAJuO9GMErN8EbkH5IYG1mbvuNWr4d13c9t27x74YdX6c2hqgmefhU8+gcZGWLUqt/3AgcH3i8s+1WlwR8SvgONNSXnqBMd8C/hWL+qyQejAAfiXf8kNf7S05MZwP/yw2FWdfGvXwg9+AO+9l/s5bN068H9RWfd0+Y+TZoW2dSvcfXexqyi+e+6BNWuKXYX1Zw5uM7M8o0ePpqamhvnz57N69WqO/ZDikSNH2Lp1a5Gqy3Fwm9mgV15ezpgxY/jqV7/KlClTmDdvHkOGDKHlOGNUu3fv5vHHH28X6Dt37uTee+9tt39TUxN79uzp03od3GY26JSUlDBs2DBqa2u5/vrrufrqqzn//PMZNWoUQ4Z8Otkuf7nVqaeeyle+8pV27c3NzXz9619v175r1y4eeOCB47b/8DjzWVtaWjhy5MgJ63dwm9mgIImRI0fyhS98gYsvvphrrrmGsrIyxo0b1yevX1JSQnl5ebv28vJyvvnNb7ZrP3r0KHfddVe79h07drBixQqeeOKJDs/l4DazAa26upqZM2fypS99ialTp9LQ0HDcK+mTrbS0lNra9t8GUltby3333cfrr7/e4bEObjMbcCZOnEhdXR3f+MY3mDBhApMmTSp2SX3KwW1mA8KECRNoaGjg1ltvZcaMGZx55plIx/1W1OQ5uM0sSSUlJVRVVTFnzhwuv/xyFi1aREVFBSWD4OsUHdxmlozW2SCLFy9m6tSpLF26lLKyMsrKyopd2knl4Dazfk0SI0aMYNGiRcyePZuFCxcybtw4hg0bVuzSisbBbWb9UnV1NRdccAE33ngj559/fr+ZDdIfOLjNrOgmT57MmDFjALjiiiuYN28elZWVTG69o4R9hoPbzE6a+vp6RowYAcDcuXOZN28eAJdddhlVVVVt+w3U2SB9xcFtZn2uoqKibQz6sssu44orrgDgmmuu4bTTTgNyHyf30EfPOLjNrMfKysrapt9deOGFXHbZZQAsXryY+vrcHQyHDRs2qP+QWAgObjPrktar46lTp3LBBRcAcPPNN3PWWWcBMGrUqLZxaissB7eZHdfZZ5/d9lHxuro6br/9dgCqqqqorq4uZmmDXqfBLWk48EugLNv/8Yi4K7sR8KNAFfAKcENEfCKpDHgY+APgI+D6iNhSoPrNrIcaGho+88145557LkuXLm1br62tbbuatv6lK1fcR4C5EXFAUinwK0n/CvwZ8N2IeFTSvcBSYEX2vCcizpG0GPg2cH2B6jezY1RUVHDqqad+pu3zn/982/hzq5kzZ7b7djrP5khDV24WHMCBbLU0ewQwF/gfWftDwN3kgnthtgzwOPB9SYpjbxdhZt2WP1uj1a233kpNTU3b+pQpU5g9e/Zn9pHkGRwDSJfGuCWVkBsOOQf4e+AdYG9ENGW7bAdaf3XXAtsAIqJJ0j5ywym/68O6LVElJSUdfq/E888/z6hRo9rdEmow2bJlC/v27WPkyJEsWLCg3QdQFi1a1G74Yvjw4YPii5XsU10K7ohoBqZJGgv8FDivtyeWtAxY1tvXsd6R1KN/Hk+dOpULL7yw28edc8453HDDDd0+bjA65ZRT2j6sYpavW7NKImKvpOeAi4CxkoZmV911QGO2WyNQD2yXNBQ4hdwfKY99rZXASgBJg/cSq49Nnz6dM844o8v7X3fddcyYMaPb56msrGw3jmpmJ0dXZpWMB45moT0CuIrcHxyfAxaRm1lyE9B6g7RV2foL2fZnPb7dsTFjxjB16tQOt3/xi19smzPbFeecc44D1WyA68oVdw3wUDbOPQR4LCKelLQeeFTSN4HXgPuy/e8D/lHSJmA3sLgAdSfnzDPPZOTIkQBcddVVbR8Brqys5NJLLy1maWaWmK7MKnkDmH6c9neBWcdpPwx8sU+qS1BVVRWlpaWUl5ezfPlySktLgdx3NLReCfsv/GbWG/3ik5OSkppJUFJSwvDhw4Hcx3xvueWWtoBesmQJtbW1bV/+7nmxZtbX+kVwT5w4kSuvvPKE+zz11FO8//77HW6PCFpaWvq0riFDhrQF7yWXXMLnPvc5IPcJsyVLlgCf3vfOV9BmdrL0i+AeO3Ys99577wn32bZtG4cOHepw+8GDB/nOd77D4cOHO9xn586dvPDCCx1uHzVqFFdeeWXbFLnly5e3fSdDdXU1Y8eO7aQnZmaF1y+CuytavyLyRB555JETbt+zZw8bNmzocPuIESOYNm2ahzfMrF9LJrj7QkVFBRdffHGxyzAz6xUPzJqZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJabT4JY0XNLLkl6X9Jakv8zaH5S0WdLa7DEta5ek70naJOkNSd2/hbiZmXWoK1/regSYGxEHJJUCv5L0r9m2r0fE48fsfzXQkD0uBFZkz2Zm1gc6veKOnAPZamn2ONENIhcCD2fHvQiMlVTT+1LNzAy6OMYtqUTSWmAX8HREvJRt+lY2HPJdSWVZWy2wLe/w7VmbmZn1gS4Fd0Q0R8Q0oA6YJWkKcCdwHnABUAl8ozsnlrRM0hpJaz788MNulm1mNnh1a1ZJROwFngPmR8SObDjkCPAAMCvbrRHIv0FkXdZ27GutjIiZETFz/PjxPavezGwQ6sqskvGSxmbLI4CrgN+0jlsrd2fda4F12SGrgBuz2SWzgX0RsaMg1ZuZDUJdmVVSAzwkqYRc0D8WEU9KelbSeEDAWuDWbP+ngAXAJuAQcHPfl21mNnh1GtwR8QYw/TjtczvYP4Dbel+amZkdjz85aWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliHNxmZolxcJuZJcbBbWaWGAe3mVliFBHFrgFJ+4GNxa6jQMYBvyt2EQUwUPsFA7dv7ldazoyI8cfbMPRkV9KBjRExs9hFFIKkNQOxbwO1XzBw++Z+DRweKjEzS4yD28wsMf0luFcWu4ACGqh9G6j9goHbN/drgOgXf5w0M7Ou6y9X3GZm1kVFD25J8yVtlLRJ0h3Frqe7JN0vaZekdXltlZKelvR29lyRtUvS97K+viFpRvEqPzFJ9ZKek7Re0luSvpa1J903ScMlvSzp9axff5m1nyXppaz+f5Y0LGsvy9Y3ZdsnFLP+zkgqkfSapCez9YHSry2S3pS0VtKarC3p92JvFDW4JZUAfw9cDUwClkiaVMyaeuBBYP4xbXcAz0REA/BMtg65fjZkj2XAipNUY080AX8eEZOA2cBt2X+b1Pt2BJgbEb8PTAPmS5oNfBv4bkScA+wBlmb7LwX2ZO3fzfbrz74GbMhbHyj9Arg8IqblTf1L/b3YcxFRtAdwEbA6b/1O4M5i1tTDfkwA1uWtbwRqsuUacvPUAf4BWHK8/fr7A3gCuGog9Q0YCbwKXEjuAxxDs/a29yWwGrgoWx6a7adi195Bf+rIBdhc4ElAA6FfWY1bgHHHtA2Y92J3H8UeKqkFtuWtb8/aUlcdETuy5Q+A6mw5yf5m/4yeDrzEAOhbNpywFtgFPA28A+yNiKZsl/za2/qVbd8HVJ3cirvsb4H/BbRk61UMjH4BBPDvkl6RtCxrS/692FP95ZOTA1ZEhKRkp+5IGg38GFgeER9LatuWat8iohmYJmks8FPgvCKX1GuSrgF2RcQrkuYUu54CuDQiGiWdCjwt6Tf5G1N9L/ZUsa+4G4H6vPW6rC11OyXVAGTPu7L2pPorqZRcaP8oIn6SNQ+IvgFExF7gOXJDCGMltV7I5Nfe1q9s+ynARye51K64BPjvkrYAj5IbLvk70u8XABHRmD3vIvfLdhYD6L3YXcUO7l8DDdlfvocBi4FVRa6pL6wCbsqWbyI3PtzafmP2V+/ZwL68f+r1K8pdWt8HbIiIe/I2Jd03SeOzK20kjSA3br+BXIAvynY7tl+t/V0EPBvZwGl/EhF3RkRdREwg9//RsxHxP0m8XwCSRkkqb10G5gHrSPy92CvFHmQHFgC/JTfO+L+LXU8P6n8E2AEcJTeWtpTcWOEzwNvAz4HKbF+Rm0XzDvAmMLPY9Z+gX5eSG1d8A1ibPRak3jfgfOC1rF/rgP+TtU8EXgY2Af8PKMvah2frm7LtE4vdhy70cQ7w5EDpV9aH17PHW605kfp7sTcPf3LSzCwxxR4qMTOzbnJwm5klxsFtZpYYB7eZWWIc3GZmiXFwm5klxsFtZpYYB7eZWWL+P8eEF+CNkxTNAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "tags": [], + "needs_background": "light" + } + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TjFBWwQP1hVe" + }, + "source": [ + "# 你的成績" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "GpJpZz3Wbm0X", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "outputId": "f1b08157-bec6-4c5a-8021-482f719b4ade" + }, + "source": [ + "print(f\"Your final reward is : %.2f\"%np.mean(test_total_reward))" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Your final reward is : -30.32\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wUBtYXG2eaqf" + }, + "source": [ + "## 參考資料\n", + "\n", + "以下是一些有用的參考資料。\n", + "建議同學們實做前，可以先參考第一則連結的上課影片。\n", + "在影片的最後有提到兩個有用的 Tips，這對於本次作業的實做非常有幫助。\n", + "\n", + "- [DRL Lecture 1: Policy Gradient (Review)](https://youtu.be/z95ZYgPgXOY)\n", + "- [ML Lecture 23-3: Reinforcement Learning (including Q-learning) start at 30:00](https://youtu.be/2-JNBzCq77c?t=1800)\n", + "- [Lecture 7: Policy Gradient, David Silver](http://www0.cs.ucl.ac.uk/staff/d.silver/web/Teaching_files/pg.pdf)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cGqP2EU1joWM" + }, + "source": [ + "" + ] + } + ] +} \ No newline at end of file diff --git a/范例/HW12/HW12_ZH.pdf b/范例/HW12/HW12_ZH.pdf new file mode 100644 index 0000000..e2c7481 Binary files /dev/null and b/范例/HW12/HW12_ZH.pdf differ