{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9c5c18a1",
   "metadata": {},
   "source": [
    "* Pong V4: https://gymnasium.farama.org/environments/atari/pong/\n",
    "* JAX installation: https://github.com/google/jax#installation\n",
    "* Proximal Policy Optimization: https://coax.readthedocs.io/en/latest/examples/stubs/ppo.html\n",
    "* Original: https://coax.readthedocs.io/en/latest/examples/atari/ppo.html"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2df4fa0b",
   "metadata": {},
   "source": [
    "# Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "df27797f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-27T23:26:44.801897Z",
     "iopub.status.busy": "2023-11-27T23:26:44.799632Z",
     "iopub.status.idle": "2023-11-27T23:26:46.272580Z",
     "shell.execute_reply": "2023-11-27T23:26:46.271931Z",
     "shell.execute_reply.started": "2023-11-27T23:26:44.801844Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/efren/.pyenv/versions/3.11.6/envs/pong/lib/python3.11/site-packages/flax/configurations.py:42: DeprecationWarning: jax.config.define_bool_state is deprecated. Please use other libraries for configuration instead.\n",
      "  return jax_config.define_bool_state('flax_' + name, default, help)\n",
      "/home/efren/.pyenv/versions/3.11.6/envs/pong/lib/python3.11/site-packages/flax/configurations.py:42: DeprecationWarning: jax.config.define_bool_state is deprecated. Please use other libraries for configuration instead.\n",
      "  return jax_config.define_bool_state('flax_' + name, default, help)\n",
      "/home/efren/.pyenv/versions/3.11.6/envs/pong/lib/python3.11/site-packages/flax/configurations.py:42: DeprecationWarning: jax.config.define_bool_state is deprecated. Please use other libraries for configuration instead.\n",
      "  return jax_config.define_bool_state('flax_' + name, default, help)\n",
      "/home/efren/.pyenv/versions/3.11.6/envs/pong/lib/python3.11/site-packages/flax/configurations.py:42: DeprecationWarning: jax.config.define_bool_state is deprecated. Please use other libraries for configuration instead.\n",
      "  return jax_config.define_bool_state('flax_' + name, default, help)\n",
      "/home/efren/.pyenv/versions/3.11.6/envs/pong/lib/python3.11/site-packages/flax/configurations.py:42: DeprecationWarning: jax.config.define_bool_state is deprecated. Please use other libraries for configuration instead.\n",
      "  return jax_config.define_bool_state('flax_' + name, default, help)\n",
      "/home/efren/.pyenv/versions/3.11.6/envs/pong/lib/python3.11/site-packages/flax/configurations.py:42: DeprecationWarning: jax.config.define_bool_state is deprecated. Please use other libraries for configuration instead.\n",
      "  return jax_config.define_bool_state('flax_' + name, default, help)\n"
     ]
    }
   ],
   "source": [
    "import gymnasium\n",
    "import jax\n",
    "import coax\n",
    "import haiku\n",
    "\n",
    "from jax             import numpy\n",
    "from matplotlib      import pyplot\n",
    "from optax           import adam\n",
    "from os              import environ\n",
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a0272f8-5939-444e-b8fa-7e62ddb8c609",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-27T21:56:37.199046Z",
     "iopub.status.busy": "2023-11-27T21:56:37.198632Z",
     "iopub.status.idle": "2023-11-27T21:56:37.201551Z",
     "shell.execute_reply": "2023-11-27T21:56:37.201190Z",
     "shell.execute_reply.started": "2023-11-27T21:56:37.199011Z"
    }
   },
   "source": [
    "## Environment Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "68fde155-7757-4110-8ff5-1cfd9d91c7d4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-27T23:26:46.273054Z",
     "iopub.status.busy": "2023-11-27T23:26:46.272883Z",
     "iopub.status.idle": "2023-11-27T23:26:46.275256Z",
     "shell.execute_reply": "2023-11-27T23:26:46.274905Z",
     "shell.execute_reply.started": "2023-11-27T23:26:46.273044Z"
    }
   },
   "outputs": [],
   "source": [
    "environ[\"JAX_PLATFORM_NAME\"]              = \"gpu\" # tell JAX to use GPU\n",
    "environ[\"XLA_PYTHON_CLIENT_MEM_FRACTION\"] = \"0.1\" # don't use all gpu mem\n",
    "environ[\"TF_CPP_MIN_LOG_LEVEL\"]           = \"3\"   # tell XLA to be quiet"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4876213b",
   "metadata": {},
   "source": [
    "# Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "092108a6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-27T23:26:46.275742Z",
     "iopub.status.busy": "2023-11-27T23:26:46.275632Z",
     "iopub.status.idle": "2023-11-27T23:26:46.370670Z",
     "shell.execute_reply": "2023-11-27T23:26:46.370130Z",
     "shell.execute_reply.started": "2023-11-27T23:26:46.275734Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A.L.E: Arcade Learning Environment (version 0.8.1+53f58b7)\n",
      "[Powered by Stella]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((array([[ 52,  52,  52, ...,  87,  87,  87],\n",
       "         [ 87,  87,  87, ...,  87,  87,  87],\n",
       "         [ 87,  87,  87, ...,  87,  87,  87],\n",
       "         ...,\n",
       "         [236, 236, 236, ..., 236, 236, 236],\n",
       "         [236, 236, 236, ..., 236, 236, 236],\n",
       "         [236, 236, 236, ..., 236, 236, 236]], dtype=uint8),\n",
       "  array([[ 52,  52,  52, ...,  87,  87,  87],\n",
       "         [ 87,  87,  87, ...,  87,  87,  87],\n",
       "         [ 87,  87,  87, ...,  87,  87,  87],\n",
       "         ...,\n",
       "         [236, 236, 236, ..., 236, 236, 236],\n",
       "         [236, 236, 236, ..., 236, 236, 236],\n",
       "         [236, 236, 236, ..., 236, 236, 236]], dtype=uint8),\n",
       "  array([[ 52,  52,  52, ...,  87,  87,  87],\n",
       "         [ 87,  87,  87, ...,  87,  87,  87],\n",
       "         [ 87,  87,  87, ...,  87,  87,  87],\n",
       "         ...,\n",
       "         [236, 236, 236, ..., 236, 236, 236],\n",
       "         [236, 236, 236, ..., 236, 236, 236],\n",
       "         [236, 236, 236, ..., 236, 236, 236]], dtype=uint8)),\n",
       " {'lives': 0, 'episode_frame_number': 20, 'frame_number': 20})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "name = \"pong\"\n",
    "\n",
    "environment = gymnasium.make('PongNoFrameskip-v4', render_mode='rgb_array')\n",
    "environment = gymnasium.wrappers.AtariPreprocessing(environment)\n",
    "environment = coax.wrappers.FrameStacking(environment, num_frames=3)\n",
    "environment = gymnasium.wrappers.TimeLimit(environment, max_episode_steps=108000 // 3)\n",
    "environment = coax.wrappers.TrainMonitor(environment, name=name, tensorboard_dir=f\"./data/tensorboard/{name}\")\n",
    "\n",
    "environment.reset()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a2a2b8f",
   "metadata": {},
   "source": [
    "## Possible actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "baa95bb6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-27T23:26:46.371450Z",
     "iopub.status.busy": "2023-11-27T23:26:46.371376Z",
     "iopub.status.idle": "2023-11-27T23:26:46.374222Z",
     "shell.execute_reply": "2023-11-27T23:26:46.373867Z",
     "shell.execute_reply.started": "2023-11-27T23:26:46.371443Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6 possible actions: ['NOOP', 'FIRE', 'RIGHT', 'LEFT', 'RIGHTFIRE', 'LEFTFIRE']\n"
     ]
    }
   ],
   "source": [
    "actions  = environment.action_space.n\n",
    "meanings = environment.unwrapped.get_action_meanings()\n",
    "\n",
    "print(f\"{actions} possible actions: {meanings}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a412017e-b1b1-4d03-8adb-ac28dd74b5eb",
   "metadata": {},
   "source": [
    "# Support Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6274ef74-a283-48cd-a74e-469e3ae530d3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-27T23:26:46.374560Z",
     "iopub.status.busy": "2023-11-27T23:26:46.374493Z",
     "iopub.status.idle": "2023-11-27T23:26:46.377311Z",
     "shell.execute_reply": "2023-11-27T23:26:46.377004Z",
     "shell.execute_reply.started": "2023-11-27T23:26:46.374553Z"
    }
   },
   "outputs": [],
   "source": [
    "def shared(S, is_training):\n",
    "    seq = haiku.Sequential([\n",
    "        coax.utils.diff_transform,\n",
    "        haiku.Conv2D(16, kernel_shape=8, stride=4), jax.nn.relu,\n",
    "        haiku.Conv2D(32, kernel_shape=4, stride=2), jax.nn.relu,\n",
    "        haiku.Flatten(),\n",
    "    ])\n",
    "    X = numpy.stack(S, axis=-1) / 255.\n",
    "    return seq(X)\n",
    "\n",
    "\n",
    "def func_pi(S, is_training):\n",
    "    logits = haiku.Sequential((\n",
    "        haiku.Linear(256), jax.nn.relu,\n",
    "        haiku.Linear(environment.action_space.n, w_init=numpy.zeros),\n",
    "    ))\n",
    "    X = shared(S, is_training)\n",
    "    return {'logits': logits(X)}\n",
    "\n",
    "\n",
    "def func_v(S, is_training):\n",
    "    value = haiku.Sequential((\n",
    "        haiku.Linear(256), jax.nn.relu,\n",
    "        haiku.Linear(1, w_init=numpy.zeros), numpy.ravel\n",
    "    ))\n",
    "    X = shared(S, is_training)\n",
    "    return value(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57f66abe-c7c6-452f-ae5e-1459a1f42688",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-27T22:48:58.667049Z",
     "iopub.status.busy": "2023-11-27T22:48:58.666635Z",
     "iopub.status.idle": "2023-11-27T22:48:58.673072Z",
     "shell.execute_reply": "2023-11-27T22:48:58.671381Z",
     "shell.execute_reply.started": "2023-11-27T22:48:58.667017Z"
    }
   },
   "source": [
    "# Function Approximators"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "25031e45-d381-4e3f-bda6-655f9835e58f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-27T23:26:46.377675Z",
     "iopub.status.busy": "2023-11-27T23:26:46.377611Z",
     "iopub.status.idle": "2023-11-27T23:26:50.164350Z",
     "shell.execute_reply": "2023-11-27T23:26:50.163850Z",
     "shell.execute_reply.started": "2023-11-27T23:26:46.377669Z"
    }
   },
   "outputs": [],
   "source": [
    "pi = coax.Policy(func_pi, environment)\n",
    "v  = coax.V(func_v, environment)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d049ed3-d80e-4d4c-b51c-aaabaa52b798",
   "metadata": {},
   "source": [
    "# Target Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6fb71c60-0855-48c0-a5f0-dfed66b64f1f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-27T23:26:50.164689Z",
     "iopub.status.busy": "2023-11-27T23:26:50.164610Z",
     "iopub.status.idle": "2023-11-27T23:26:50.166742Z",
     "shell.execute_reply": "2023-11-27T23:26:50.166451Z",
     "shell.execute_reply.started": "2023-11-27T23:26:50.164682Z"
    }
   },
   "outputs": [],
   "source": [
    "pi_behavior = pi.copy()\n",
    "v_targ      = v.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf0d0c61-2cb7-4e72-9a85-978beaa53fcc",
   "metadata": {},
   "source": [
    "# Policy Regularizer (Avoid Premature Exploitation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f4d82b64-980c-46a4-9dde-399e54f4ff64",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-27T23:26:50.167166Z",
     "iopub.status.busy": "2023-11-27T23:26:50.167029Z",
     "iopub.status.idle": "2023-11-27T23:26:50.168814Z",
     "shell.execute_reply": "2023-11-27T23:26:50.168635Z",
     "shell.execute_reply.started": "2023-11-27T23:26:50.167158Z"
    }
   },
   "outputs": [],
   "source": [
    "entropy = coax.regularizers.EntropyRegularizer(pi, beta=0.001)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "706d4d5c-d9a6-4b21-a08e-9ea1089a531c",
   "metadata": {},
   "source": [
    "# Updaters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b8e0169b-168c-49bc-8427-afd9b9730bbe",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-27T23:26:50.169140Z",
     "iopub.status.busy": "2023-11-27T23:26:50.169047Z",
     "iopub.status.idle": "2023-11-27T23:26:50.245330Z",
     "shell.execute_reply": "2023-11-27T23:26:50.245083Z",
     "shell.execute_reply.started": "2023-11-27T23:26:50.169133Z"
    }
   },
   "outputs": [],
   "source": [
    "simpletd = coax.td_learning.SimpleTD(v, v_targ, optimizer=adam(3e-4))\n",
    "ppo_clip = coax.policy_objectives.PPOClip(pi, regularizer=entropy, optimizer=adam(3e-4))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "360b2db3-fb04-4621-bef6-4a8ba574a06f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-27T22:50:33.890963Z",
     "iopub.status.busy": "2023-11-27T22:50:33.890531Z",
     "iopub.status.idle": "2023-11-27T22:50:33.896271Z",
     "shell.execute_reply": "2023-11-27T22:50:33.895067Z",
     "shell.execute_reply.started": "2023-11-27T22:50:33.890928Z"
    }
   },
   "source": [
    "# Reward Tracer and Replay Buffer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "83290b28-fd4c-4511-8c15-1dd12d899b2e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-27T23:26:50.245729Z",
     "iopub.status.busy": "2023-11-27T23:26:50.245651Z",
     "iopub.status.idle": "2023-11-27T23:26:50.247560Z",
     "shell.execute_reply": "2023-11-27T23:26:50.247348Z",
     "shell.execute_reply.started": "2023-11-27T23:26:50.245722Z"
    }
   },
   "outputs": [],
   "source": [
    "tracer = coax.reward_tracing.NStep(n=5, gamma=0.99)\n",
    "buffer = coax.experience_replay.SimpleReplayBuffer(capacity=256)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e438eb96-d5a1-4fda-83c6-af717686b97d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-27T22:52:03.161776Z",
     "iopub.status.busy": "2023-11-27T22:52:03.160449Z",
     "iopub.status.idle": "2023-11-27T22:52:03.164469Z",
     "shell.execute_reply": "2023-11-27T22:52:03.163946Z",
     "shell.execute_reply.started": "2023-11-27T22:52:03.161723Z"
    }
   },
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "387e1e90-c6ff-4110-94b7-58fa00e4a90c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-27T23:26:50.247937Z",
     "iopub.status.busy": "2023-11-27T23:26:50.247821Z",
     "iopub.status.idle": "2023-11-28T02:19:31.421578Z",
     "shell.execute_reply": "2023-11-28T02:19:31.421255Z",
     "shell.execute_reply.started": "2023-11-27T23:26:50.247929Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:TrainMonitor:ep: 1,\tT: 1,\tG: 0,\tavg_r: nan,\tavg_G: 0,\tt: 0,\tdt: nanms\n",
      "WARNING:tensorboardX.x2num:NaN or Inf found in input tensor.\n",
      "WARNING:tensorboardX.x2num:NaN or Inf found in input tensor.\n",
      "INFO:TrainMonitor:ep: 2,\tT: 933,\tG: -20,\tavg_r: -0.0215,\tavg_G: -20,\tt: 931,\tdt: 6.052ms,\tSimpleTD/loss: 0.0499,\tPPOClip/EntropyRegularizer/entropy: 1.79,\tPPOClip/loss: 0.0104\n",
      "INFO:TrainMonitor:ep: 3,\tT: 1,758,\tG: -21,\tavg_r: -0.0255,\tavg_G: -20.5,\tt: 824,\tdt: 2.822ms,\tSimpleTD/loss: 0.0456,\tPPOClip/EntropyRegularizer/entropy: 1.78,\tPPOClip/loss: -0.000834\n",
      "INFO:TrainMonitor:ep: 4,\tT: 2,788,\tG: -20,\tavg_r: -0.0194,\tavg_G: -20.3,\tt: 1029,\tdt: 2.838ms,\tSimpleTD/loss: 0.0168,\tPPOClip/EntropyRegularizer/entropy: 1.79,\tPPOClip/loss: 0.000254\n",
      "INFO:TrainMonitor:ep: 5,\tT: 3,629,\tG: -20,\tavg_r: -0.0238,\tavg_G: -20.2,\tt: 840,\tdt: 2.921ms,\tSimpleTD/loss: 0.00722,\tPPOClip/EntropyRegularizer/entropy: 1.78,\tPPOClip/loss: -0.00171\n",
      "INFO:TrainMonitor:ep: 6,\tT: 4,607,\tG: -20,\tavg_r: -0.0205,\tavg_G: -20.2,\tt: 977,\tdt: 2.795ms,\tSimpleTD/loss: 0.0162,\tPPOClip/EntropyRegularizer/entropy: 1.78,\tPPOClip/loss: -0.00592\n",
      "INFO:TrainMonitor:ep: 7,\tT: 5,787,\tG: -18,\tavg_r: -0.0153,\tavg_G: -19.8,\tt: 1179,\tdt: 2.881ms,\tSimpleTD/loss: 0.0164,\tPPOClip/EntropyRegularizer/entropy: 1.79,\tPPOClip/loss: -0.00534\n",
      "INFO:TrainMonitor:ep: 8,\tT: 6,736,\tG: -21,\tavg_r: -0.0222,\tavg_G: -20,\tt: 948,\tdt: 2.885ms,\tSimpleTD/loss: 0.00942,\tPPOClip/EntropyRegularizer/entropy: 1.78,\tPPOClip/loss: 0.000203\n",
      "INFO:TrainMonitor:ep: 9,\tT: 7,743,\tG: -20,\tavg_r: -0.0199,\tavg_G: -20,\tt: 1006,\tdt: 2.845ms,\tSimpleTD/loss: 0.0144,\tPPOClip/EntropyRegularizer/entropy: 1.79,\tPPOClip/loss: -0.00313\n",
      "INFO:TrainMonitor:ep: 10,\tT: 8,564,\tG: -21,\tavg_r: -0.0256,\tavg_G: -20.1,\tt: 820,\tdt: 2.830ms,\tSimpleTD/loss: 0.00292,\tPPOClip/EntropyRegularizer/entropy: 1.79,\tPPOClip/loss: -0.000858\n",
      "INFO:TrainMonitor:ep: 11,\tT: 9,479,\tG: -20,\tavg_r: -0.0219,\tavg_G: -20.1,\tt: 914,\tdt: 2.783ms,\tSimpleTD/loss: 0.00498,\tPPOClip/EntropyRegularizer/entropy: 1.79,\tPPOClip/loss: -0.000736\n",
      "/home/efren/.pyenv/versions/3.11.6/envs/pong/lib/python3.11/site-packages/gymnasium/utils/passive_env_checker.py:335: UserWarning: \u001b[33mWARN: No render fps was declared in the environment (env.metadata['render_fps'] is None or not defined), rendering may occur at inconsistent fps.\u001b[0m\n",
      "  logger.warn(\n",
      "INFO:TrainMonitor:ep: 12,\tT: 10,243,\tG: -21,\tavg_r: -0.0275,\tavg_G: -20.2,\tt: 763,\tdt: 6.742ms,\tSimpleTD/loss: 0.0049,\tPPOClip/EntropyRegularizer/entropy: 1.78,\tPPOClip/loss: -0.00253\n",
      "INFO:TrainMonitor:ep: 13,\tT: 11,208,\tG: -20,\tavg_r: -0.0207,\tavg_G: -20.2,\tt: 964,\tdt: 2.869ms,\tSimpleTD/loss: 0.00715,\tPPOClip/EntropyRegularizer/entropy: 1.79,\tPPOClip/loss: -0.000982\n",
      "INFO:TrainMonitor:ep: 14,\tT: 12,105,\tG: -20,\tavg_r: -0.0223,\tavg_G: -20.2,\tt: 896,\tdt: 2.886ms,\tSimpleTD/loss: 0.00845,\tPPOClip/EntropyRegularizer/entropy: 1.79,\tPPOClip/loss: -0.00114\n",
      "INFO:TrainMonitor:ep: 15,\tT: 13,013,\tG: -21,\tavg_r: -0.0232,\tavg_G: -20.2,\tt: 907,\tdt: 2.794ms,\tSimpleTD/loss: 0.00682,\tPPOClip/EntropyRegularizer/entropy: 1.79,\tPPOClip/loss: 0.00102\n",
      "INFO:TrainMonitor:ep: 16,\tT: 13,925,\tG: -21,\tavg_r: -0.0231,\tavg_G: -20.3,\tt: 911,\tdt: 2.881ms,\tSimpleTD/loss: 0.00811,\tPPOClip/EntropyRegularizer/entropy: 1.79,\tPPOClip/loss: -0.00149\n",
      "INFO:TrainMonitor:ep: 17,\tT: 14,804,\tG: -21,\tavg_r: -0.0239,\tavg_G: -20.4,\tt: 878,\tdt: 2.798ms,\tSimpleTD/loss: 0.00613,\tPPOClip/EntropyRegularizer/entropy: 1.79,\tPPOClip/loss: -0.000283\n",
      "INFO:TrainMonitor:ep: 18,\tT: 15,899,\tG: -19,\tavg_r: -0.0174,\tavg_G: -20.2,\tt: 1094,\tdt: 2.929ms,\tSimpleTD/loss: 0.0122,\tPPOClip/EntropyRegularizer/entropy: 1.78,\tPPOClip/loss: -0.00189\n",
      "INFO:TrainMonitor:ep: 19,\tT: 16,731,\tG: -20,\tavg_r: -0.0241,\tavg_G: -20.2,\tt: 831,\tdt: 2.834ms,\tSimpleTD/loss: 0.00773,\tPPOClip/EntropyRegularizer/entropy: 1.78,\tPPOClip/loss: 0.000543\n",
      "INFO:TrainMonitor:ep: 20,\tT: 17,574,\tG: -21,\tavg_r: -0.0249,\tavg_G: -20.3,\tt: 842,\tdt: 2.846ms,\tSimpleTD/loss: 0.00395,\tPPOClip/EntropyRegularizer/entropy: 1.79,\tPPOClip/loss: -0.00101\n",
      "INFO:TrainMonitor:ep: 21,\tT: 18,643,\tG: -20,\tavg_r: -0.0187,\tavg_G: -20.3,\tt: 1068,\tdt: 2.845ms,\tSimpleTD/loss: 0.0144,\tPPOClip/EntropyRegularizer/entropy: 1.78,\tPPOClip/loss: -0.00172\n",
      "INFO:TrainMonitor:ep: 22,\tT: 19,627,\tG: -20,\tavg_r: -0.0203,\tavg_G: -20.2,\tt: 983,\tdt: 2.893ms,\tSimpleTD/loss: 0.00789,\tPPOClip/EntropyRegularizer/entropy: 1.78,\tPPOClip/loss: -0.00107\n",
      "INFO:TrainMonitor:ep: 23,\tT: 20,413,\tG: -21,\tavg_r: -0.0268,\tavg_G: -20.3,\tt: 785,\tdt: 7.704ms,\tSimpleTD/loss: 0.00986,\tPPOClip/EntropyRegularizer/entropy: 1.78,\tPPOClip/loss: -0.00198\n",
      "INFO:TrainMonitor:ep: 24,\tT: 21,419,\tG: -20,\tavg_r: -0.0199,\tavg_G: -20.3,\tt: 1005,\tdt: 2.874ms,\tSimpleTD/loss: 0.00958,\tPPOClip/EntropyRegularizer/entropy: 1.78,\tPPOClip/loss: -0.00097\n",
      "INFO:TrainMonitor:ep: 25,\tT: 22,552,\tG: -18,\tavg_r: -0.0159,\tavg_G: -20.1,\tt: 1132,\tdt: 2.830ms,\tSimpleTD/loss: 0.0161,\tPPOClip/EntropyRegularizer/entropy: 1.78,\tPPOClip/loss: -0.00637\n",
      "INFO:TrainMonitor:ep: 26,\tT: 23,386,\tG: -21,\tavg_r: -0.0252,\tavg_G: -20.2,\tt: 833,\tdt: 2.961ms,\tSimpleTD/loss: 0.0177,\tPPOClip/EntropyRegularizer/entropy: 1.78,\tPPOClip/loss: -0.00151\n",
      "INFO:TrainMonitor:ep: 27,\tT: 24,467,\tG: -20,\tavg_r: -0.0185,\tavg_G: -20.1,\tt: 1080,\tdt: 2.870ms,\tSimpleTD/loss: 0.0108,\tPPOClip/EntropyRegularizer/entropy: 1.77,\tPPOClip/loss: -0.00358\n",
      "INFO:TrainMonitor:ep: 28,\tT: 25,422,\tG: -20,\tavg_r: -0.021,\tavg_G: -20.1,\tt: 954,\tdt: 2.886ms,\tSimpleTD/loss: 0.0139,\tPPOClip/EntropyRegularizer/entropy: 1.78,\tPPOClip/loss: -0.00388\n",
      "INFO:TrainMonitor:ep: 29,\tT: 26,300,\tG: -21,\tavg_r: -0.0239,\tavg_G: -20.2,\tt: 877,\tdt: 2.812ms,\tSimpleTD/loss: 0.0121,\tPPOClip/EntropyRegularizer/entropy: 1.77,\tPPOClip/loss: -0.00236\n",
      "INFO:TrainMonitor:ep: 30,\tT: 27,301,\tG: -21,\tavg_r: -0.021,\tavg_G: -20.3,\tt: 1000,\tdt: 2.947ms,\tSimpleTD/loss: 0.00694,\tPPOClip/EntropyRegularizer/entropy: 1.77,\tPPOClip/loss: -0.00357\n",
      "INFO:TrainMonitor:ep: 31,\tT: 28,080,\tG: -21,\tavg_r: -0.027,\tavg_G: -20.4,\tt: 778,\tdt: 2.874ms,\tSimpleTD/loss: 0.00819,\tPPOClip/EntropyRegularizer/entropy: 1.77,\tPPOClip/loss: -0.00182\n",
      "INFO:TrainMonitor:ep: 32,\tT: 28,889,\tG: -21,\tavg_r: -0.026,\tavg_G: -20.4,\tt: 808,\tdt: 2.853ms,\tSimpleTD/loss: 0.00905,\tPPOClip/EntropyRegularizer/entropy: 1.77,\tPPOClip/loss: -0.00443\n",
      "INFO:TrainMonitor:ep: 33,\tT: 29,973,\tG: -20,\tavg_r: -0.0185,\tavg_G: -20.4,\tt: 1083,\tdt: 2.844ms,\tSimpleTD/loss: 0.0123,\tPPOClip/EntropyRegularizer/entropy: 1.77,\tPPOClip/loss: -0.000609\n",
      "INFO:TrainMonitor:ep: 34,\tT: 30,788,\tG: -21,\tavg_r: -0.0258,\tavg_G: -20.4,\tt: 814,\tdt: 6.672ms,\tSimpleTD/loss: 0.0113,\tPPOClip/EntropyRegularizer/entropy: 1.77,\tPPOClip/loss: -0.00387\n",
      "INFO:TrainMonitor:ep: 35,\tT: 31,696,\tG: -21,\tavg_r: -0.0232,\tavg_G: -20.5,\tt: 907,\tdt: 2.815ms,\tSimpleTD/loss: 0.00518,\tPPOClip/EntropyRegularizer/entropy: 1.77,\tPPOClip/loss: -0.00296\n",
      "INFO:TrainMonitor:ep: 36,\tT: 32,568,\tG: -20,\tavg_r: -0.023,\tavg_G: -20.4,\tt: 871,\tdt: 2.926ms,\tSimpleTD/loss: 0.0152,\tPPOClip/EntropyRegularizer/entropy: 1.77,\tPPOClip/loss: -0.00324\n",
      "INFO:TrainMonitor:ep: 37,\tT: 33,631,\tG: -21,\tavg_r: -0.0198,\tavg_G: -20.5,\tt: 1062,\tdt: 2.918ms,\tSimpleTD/loss: 0.0151,\tPPOClip/EntropyRegularizer/entropy: 1.77,\tPPOClip/loss: -0.0044\n",
      "INFO:TrainMonitor:ep: 38,\tT: 34,519,\tG: -21,\tavg_r: -0.0237,\tavg_G: -20.6,\tt: 887,\tdt: 2.813ms,\tSimpleTD/loss: 0.0123,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.00182\n",
      "INFO:TrainMonitor:ep: 39,\tT: 35,571,\tG: -19,\tavg_r: -0.0181,\tavg_G: -20.4,\tt: 1051,\tdt: 2.882ms,\tSimpleTD/loss: 0.017,\tPPOClip/EntropyRegularizer/entropy: 1.77,\tPPOClip/loss: -0.00385\n",
      "INFO:TrainMonitor:ep: 40,\tT: 36,381,\tG: -21,\tavg_r: -0.026,\tavg_G: -20.5,\tt: 809,\tdt: 2.842ms,\tSimpleTD/loss: 0.012,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.00143\n",
      "INFO:TrainMonitor:ep: 41,\tT: 37,336,\tG: -20,\tavg_r: -0.021,\tavg_G: -20.4,\tt: 954,\tdt: 2.922ms,\tSimpleTD/loss: 0.0107,\tPPOClip/EntropyRegularizer/entropy: 1.77,\tPPOClip/loss: -0.00407\n",
      "INFO:TrainMonitor:ep: 42,\tT: 38,491,\tG: -20,\tavg_r: -0.0173,\tavg_G: -20.4,\tt: 1154,\tdt: 2.881ms,\tSimpleTD/loss: 0.0151,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.0071\n",
      "INFO:TrainMonitor:ep: 43,\tT: 39,328,\tG: -20,\tavg_r: -0.0239,\tavg_G: -20.3,\tt: 836,\tdt: 2.812ms,\tSimpleTD/loss: 0.00943,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00185\n",
      "INFO:TrainMonitor:ep: 44,\tT: 40,318,\tG: -19,\tavg_r: -0.0192,\tavg_G: -20.2,\tt: 989,\tdt: 6.672ms,\tSimpleTD/loss: 0.0134,\tPPOClip/EntropyRegularizer/entropy: 1.74,\tPPOClip/loss: -0.00522\n",
      "INFO:TrainMonitor:ep: 45,\tT: 41,354,\tG: -20,\tavg_r: -0.0193,\tavg_G: -20.2,\tt: 1035,\tdt: 2.854ms,\tSimpleTD/loss: 0.016,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.00431\n",
      "INFO:TrainMonitor:ep: 46,\tT: 42,468,\tG: -19,\tavg_r: -0.0171,\tavg_G: -20.1,\tt: 1113,\tdt: 2.855ms,\tSimpleTD/loss: 0.0132,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.00394\n",
      "INFO:TrainMonitor:ep: 47,\tT: 43,516,\tG: -20,\tavg_r: -0.0191,\tavg_G: -20.1,\tt: 1047,\tdt: 2.843ms,\tSimpleTD/loss: 0.0121,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.00288\n",
      "INFO:TrainMonitor:ep: 48,\tT: 44,569,\tG: -19,\tavg_r: -0.0181,\tavg_G: -20,\tt: 1052,\tdt: 2.839ms,\tSimpleTD/loss: 0.0156,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.00424\n",
      "INFO:TrainMonitor:ep: 49,\tT: 45,620,\tG: -21,\tavg_r: -0.02,\tavg_G: -20.1,\tt: 1050,\tdt: 2.922ms,\tSimpleTD/loss: 0.0133,\tPPOClip/EntropyRegularizer/entropy: 1.77,\tPPOClip/loss: -0.00328\n",
      "INFO:TrainMonitor:ep: 50,\tT: 46,573,\tG: -20,\tavg_r: -0.021,\tavg_G: -20.1,\tt: 952,\tdt: 2.786ms,\tSimpleTD/loss: 0.0148,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.00273\n",
      "INFO:TrainMonitor:ep: 51,\tT: 47,455,\tG: -21,\tavg_r: -0.0238,\tavg_G: -20.1,\tt: 881,\tdt: 2.914ms,\tSimpleTD/loss: 0.0126,\tPPOClip/EntropyRegularizer/entropy: 1.77,\tPPOClip/loss: -0.00293\n",
      "INFO:TrainMonitor:ep: 52,\tT: 48,358,\tG: -20,\tavg_r: -0.0222,\tavg_G: -20.1,\tt: 902,\tdt: 2.792ms,\tSimpleTD/loss: 0.0167,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.000189\n",
      "INFO:TrainMonitor:ep: 53,\tT: 49,519,\tG: -19,\tavg_r: -0.0164,\tavg_G: -20,\tt: 1160,\tdt: 2.894ms,\tSimpleTD/loss: 0.0131,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.00562\n",
      "INFO:TrainMonitor:ep: 54,\tT: 50,432,\tG: -20,\tavg_r: -0.0219,\tavg_G: -20,\tt: 912,\tdt: 6.159ms,\tSimpleTD/loss: 0.015,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00516\n",
      "INFO:TrainMonitor:ep: 55,\tT: 51,442,\tG: -21,\tavg_r: -0.0208,\tavg_G: -20.1,\tt: 1009,\tdt: 2.874ms,\tSimpleTD/loss: 0.0161,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.00418\n",
      "INFO:TrainMonitor:ep: 56,\tT: 52,292,\tG: -21,\tavg_r: -0.0247,\tavg_G: -20.2,\tt: 849,\tdt: 2.934ms,\tSimpleTD/loss: 0.0108,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.00329\n",
      "INFO:TrainMonitor:ep: 57,\tT: 53,099,\tG: -21,\tavg_r: -0.0261,\tavg_G: -20.3,\tt: 806,\tdt: 2.819ms,\tSimpleTD/loss: 0.0107,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.00311\n",
      "INFO:TrainMonitor:ep: 58,\tT: 54,122,\tG: -21,\tavg_r: -0.0205,\tavg_G: -20.4,\tt: 1022,\tdt: 2.865ms,\tSimpleTD/loss: 0.0112,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00558\n",
      "INFO:TrainMonitor:ep: 59,\tT: 55,044,\tG: -20,\tavg_r: -0.0217,\tavg_G: -20.3,\tt: 921,\tdt: 2.794ms,\tSimpleTD/loss: 0.0164,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.0036\n",
      "INFO:TrainMonitor:ep: 60,\tT: 56,084,\tG: -20,\tavg_r: -0.0192,\tavg_G: -20.3,\tt: 1039,\tdt: 2.846ms,\tSimpleTD/loss: 0.0137,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00413\n",
      "INFO:TrainMonitor:ep: 61,\tT: 56,936,\tG: -21,\tavg_r: -0.0247,\tavg_G: -20.4,\tt: 851,\tdt: 2.920ms,\tSimpleTD/loss: 0.00943,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.00328\n",
      "INFO:TrainMonitor:ep: 62,\tT: 57,958,\tG: -19,\tavg_r: -0.0186,\tavg_G: -20.2,\tt: 1021,\tdt: 2.903ms,\tSimpleTD/loss: 0.0162,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00483\n",
      "INFO:TrainMonitor:ep: 63,\tT: 58,983,\tG: -19,\tavg_r: -0.0186,\tavg_G: -20.1,\tt: 1024,\tdt: 2.848ms,\tSimpleTD/loss: 0.0143,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.00548\n",
      "INFO:TrainMonitor:ep: 64,\tT: 60,050,\tG: -19,\tavg_r: -0.0178,\tavg_G: -20,\tt: 1066,\tdt: 5.825ms,\tSimpleTD/loss: 0.0155,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00457\n",
      "INFO:TrainMonitor:ep: 65,\tT: 60,934,\tG: -20,\tavg_r: -0.0227,\tavg_G: -20,\tt: 883,\tdt: 2.817ms,\tSimpleTD/loss: 0.0121,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00343\n",
      "INFO:TrainMonitor:ep: 66,\tT: 61,960,\tG: -21,\tavg_r: -0.0205,\tavg_G: -20.1,\tt: 1025,\tdt: 2.837ms,\tSimpleTD/loss: 0.0138,\tPPOClip/EntropyRegularizer/entropy: 1.74,\tPPOClip/loss: -0.00432\n",
      "INFO:TrainMonitor:ep: 67,\tT: 63,006,\tG: -19,\tavg_r: -0.0182,\tavg_G: -20,\tt: 1045,\tdt: 2.845ms,\tSimpleTD/loss: 0.0161,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00556\n",
      "INFO:TrainMonitor:ep: 68,\tT: 64,058,\tG: -21,\tavg_r: -0.02,\tavg_G: -20.1,\tt: 1051,\tdt: 2.859ms,\tSimpleTD/loss: 0.0133,\tPPOClip/EntropyRegularizer/entropy: 1.74,\tPPOClip/loss: -0.00458\n",
      "INFO:TrainMonitor:ep: 69,\tT: 65,078,\tG: -20,\tavg_r: -0.0196,\tavg_G: -20.1,\tt: 1019,\tdt: 2.854ms,\tSimpleTD/loss: 0.0131,\tPPOClip/EntropyRegularizer/entropy: 1.74,\tPPOClip/loss: -0.00435\n",
      "INFO:TrainMonitor:ep: 70,\tT: 66,521,\tG: -16,\tavg_r: -0.0111,\tavg_G: -19.7,\tt: 1442,\tdt: 2.889ms,\tSimpleTD/loss: 0.0177,\tPPOClip/EntropyRegularizer/entropy: 1.74,\tPPOClip/loss: -0.00587\n",
      "INFO:TrainMonitor:ep: 71,\tT: 67,450,\tG: -20,\tavg_r: -0.0216,\tavg_G: -19.7,\tt: 928,\tdt: 2.891ms,\tSimpleTD/loss: 0.0151,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00566\n",
      "INFO:TrainMonitor:ep: 72,\tT: 68,614,\tG: -18,\tavg_r: -0.0155,\tavg_G: -19.5,\tt: 1163,\tdt: 2.823ms,\tSimpleTD/loss: 0.0152,\tPPOClip/EntropyRegularizer/entropy: 1.74,\tPPOClip/loss: -0.00473\n",
      "INFO:TrainMonitor:ep: 73,\tT: 69,761,\tG: -20,\tavg_r: -0.0175,\tavg_G: -19.6,\tt: 1146,\tdt: 2.902ms,\tSimpleTD/loss: 0.0171,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00471\n",
      "INFO:TrainMonitor:ep: 74,\tT: 70,658,\tG: -21,\tavg_r: -0.0234,\tavg_G: -19.7,\tt: 896,\tdt: 6.852ms,\tSimpleTD/loss: 0.00994,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00555\n",
      "INFO:TrainMonitor:ep: 75,\tT: 71,706,\tG: -20,\tavg_r: -0.0191,\tavg_G: -19.7,\tt: 1047,\tdt: 2.863ms,\tSimpleTD/loss: 0.0164,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.00225\n",
      "INFO:TrainMonitor:ep: 76,\tT: 72,652,\tG: -21,\tavg_r: -0.0222,\tavg_G: -19.9,\tt: 945,\tdt: 2.888ms,\tSimpleTD/loss: 0.0115,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00307\n",
      "INFO:TrainMonitor:ep: 77,\tT: 73,615,\tG: -20,\tavg_r: -0.0208,\tavg_G: -19.9,\tt: 962,\tdt: 2.903ms,\tSimpleTD/loss: 0.0122,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00613\n",
      "INFO:TrainMonitor:ep: 78,\tT: 74,579,\tG: -20,\tavg_r: -0.0208,\tavg_G: -19.9,\tt: 963,\tdt: 2.854ms,\tSimpleTD/loss: 0.011,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00455\n",
      "INFO:TrainMonitor:ep: 79,\tT: 75,389,\tG: -21,\tavg_r: -0.026,\tavg_G: -20,\tt: 809,\tdt: 2.828ms,\tSimpleTD/loss: 0.00972,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.00343\n",
      "INFO:TrainMonitor:ep: 80,\tT: 76,561,\tG: -18,\tavg_r: -0.0154,\tavg_G: -19.8,\tt: 1171,\tdt: 2.824ms,\tSimpleTD/loss: 0.0166,\tPPOClip/EntropyRegularizer/entropy: 1.74,\tPPOClip/loss: -0.0062\n",
      "INFO:TrainMonitor:ep: 81,\tT: 77,519,\tG: -21,\tavg_r: -0.0219,\tavg_G: -19.9,\tt: 957,\tdt: 2.895ms,\tSimpleTD/loss: 0.0124,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.00401\n",
      "INFO:TrainMonitor:ep: 82,\tT: 78,481,\tG: -20,\tavg_r: -0.0208,\tavg_G: -19.9,\tt: 961,\tdt: 2.874ms,\tSimpleTD/loss: 0.00849,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00454\n",
      "INFO:TrainMonitor:ep: 83,\tT: 79,643,\tG: -19,\tavg_r: -0.0164,\tavg_G: -19.8,\tt: 1161,\tdt: 2.820ms,\tSimpleTD/loss: 0.0123,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00296\n",
      "INFO:TrainMonitor:ep: 84,\tT: 80,767,\tG: -20,\tavg_r: -0.0178,\tavg_G: -19.9,\tt: 1123,\tdt: 5.811ms,\tSimpleTD/loss: 0.0179,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00635\n",
      "INFO:TrainMonitor:ep: 85,\tT: 81,615,\tG: -21,\tavg_r: -0.0248,\tavg_G: -20,\tt: 847,\tdt: 2.845ms,\tSimpleTD/loss: 0.0114,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00296\n",
      "INFO:TrainMonitor:ep: 86,\tT: 82,778,\tG: -20,\tavg_r: -0.0172,\tavg_G: -20,\tt: 1162,\tdt: 2.853ms,\tSimpleTD/loss: 0.0147,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.0065\n",
      "INFO:TrainMonitor:ep: 87,\tT: 83,660,\tG: -21,\tavg_r: -0.0238,\tavg_G: -20.1,\tt: 881,\tdt: 2.909ms,\tSimpleTD/loss: 0.0106,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.0024\n",
      "INFO:TrainMonitor:ep: 88,\tT: 84,699,\tG: -20,\tavg_r: -0.0193,\tavg_G: -20.1,\tt: 1038,\tdt: 2.863ms,\tSimpleTD/loss: 0.0127,\tPPOClip/EntropyRegularizer/entropy: 1.74,\tPPOClip/loss: -0.00457\n",
      "INFO:TrainMonitor:ep: 89,\tT: 85,595,\tG: -21,\tavg_r: -0.0235,\tavg_G: -20.2,\tt: 895,\tdt: 2.832ms,\tSimpleTD/loss: 0.00976,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00301\n",
      "INFO:TrainMonitor:ep: 90,\tT: 86,501,\tG: -21,\tavg_r: -0.0232,\tavg_G: -20.2,\tt: 905,\tdt: 2.895ms,\tSimpleTD/loss: 0.011,\tPPOClip/EntropyRegularizer/entropy: 1.74,\tPPOClip/loss: -0.00485\n",
      "INFO:TrainMonitor:ep: 91,\tT: 87,631,\tG: -20,\tavg_r: -0.0177,\tavg_G: -20.2,\tt: 1129,\tdt: 2.844ms,\tSimpleTD/loss: 0.0111,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00406\n",
      "INFO:TrainMonitor:ep: 92,\tT: 88,636,\tG: -20,\tavg_r: -0.0199,\tavg_G: -20.2,\tt: 1004,\tdt: 2.862ms,\tSimpleTD/loss: 0.0146,\tPPOClip/EntropyRegularizer/entropy: 1.73,\tPPOClip/loss: -0.00413\n",
      "INFO:TrainMonitor:ep: 93,\tT: 89,475,\tG: -21,\tavg_r: -0.0251,\tavg_G: -20.3,\tt: 838,\tdt: 2.943ms,\tSimpleTD/loss: 0.0102,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00437\n",
      "INFO:TrainMonitor:ep: 94,\tT: 90,372,\tG: -21,\tavg_r: -0.0234,\tavg_G: -20.4,\tt: 896,\tdt: 6.762ms,\tSimpleTD/loss: 0.0114,\tPPOClip/EntropyRegularizer/entropy: 1.74,\tPPOClip/loss: -0.00349\n",
      "INFO:TrainMonitor:ep: 95,\tT: 91,237,\tG: -21,\tavg_r: -0.0243,\tavg_G: -20.4,\tt: 864,\tdt: 2.936ms,\tSimpleTD/loss: 0.0101,\tPPOClip/EntropyRegularizer/entropy: 1.75,\tPPOClip/loss: -0.00497\n",
      "INFO:TrainMonitor:ep: 96,\tT: 92,062,\tG: -21,\tavg_r: -0.0255,\tavg_G: -20.5,\tt: 824,\tdt: 2.851ms,\tSimpleTD/loss: 0.00705,\tPPOClip/EntropyRegularizer/entropy: 1.76,\tPPOClip/loss: -0.00211\n",
      "INFO:TrainMonitor:ep: 97,\tT: 93,441,\tG: -18,\tavg_r: -0.0131,\tavg_G: -20.2,\tt: 1378,\tdt: 2.844ms,\tSimpleTD/loss: 0.0123,\tPPOClip/EntropyRegularizer/entropy: 1.74,\tPPOClip/loss: -0.0045\n",
      "INFO:TrainMonitor:ep: 98,\tT: 94,647,\tG: -18,\tavg_r: -0.0149,\tavg_G: -20,\tt: 1205,\tdt: 2.872ms,\tSimpleTD/loss: 0.0143,\tPPOClip/EntropyRegularizer/entropy: 1.73,\tPPOClip/loss: -0.00523\n",
      "INFO:TrainMonitor:ep: 99,\tT: 95,800,\tG: -20,\tavg_r: -0.0174,\tavg_G: -20,\tt: 1152,\tdt: 2.825ms,\tSimpleTD/loss: 0.0139,\tPPOClip/EntropyRegularizer/entropy: 1.73,\tPPOClip/loss: -0.00388\n",
      "INFO:TrainMonitor:ep: 100,\tT: 97,007,\tG: -18,\tavg_r: -0.0149,\tavg_G: -19.8,\tt: 1206,\tdt: 2.875ms,\tSimpleTD/loss: 0.0154,\tPPOClip/EntropyRegularizer/entropy: 1.74,\tPPOClip/loss: -0.00562\n",
      "INFO:TrainMonitor:ep: 101,\tT: 97,961,\tG: -20,\tavg_r: -0.021,\tavg_G: -19.8,\tt: 953,\tdt: 2.912ms,\tSimpleTD/loss: 0.0126,\tPPOClip/EntropyRegularizer/entropy: 1.72,\tPPOClip/loss: -0.00475\n",
      "INFO:TrainMonitor:ep: 102,\tT: 99,034,\tG: -20,\tavg_r: -0.0187,\tavg_G: -19.8,\tt: 1072,\tdt: 2.836ms,\tSimpleTD/loss: 0.0187,\tPPOClip/EntropyRegularizer/entropy: 1.74,\tPPOClip/loss: -0.00508\n",
      "INFO:TrainMonitor:ep: 103,\tT: 100,073,\tG: -20,\tavg_r: -0.0193,\tavg_G: -19.9,\tt: 1038,\tdt: 5.431ms,\tSimpleTD/loss: 0.0166,\tPPOClip/EntropyRegularizer/entropy: 1.73,\tPPOClip/loss: -0.00453\n",
      "INFO:TrainMonitor:ep: 104,\tT: 101,249,\tG: -18,\tavg_r: -0.0153,\tavg_G: -19.7,\tt: 1175,\tdt: 2.895ms,\tSimpleTD/loss: 0.0135,\tPPOClip/EntropyRegularizer/entropy: 1.72,\tPPOClip/loss: -0.00575\n",
      "INFO:TrainMonitor:ep: 105,\tT: 102,488,\tG: -19,\tavg_r: -0.0153,\tavg_G: -19.6,\tt: 1238,\tdt: 2.802ms,\tSimpleTD/loss: 0.0151,\tPPOClip/EntropyRegularizer/entropy: 1.72,\tPPOClip/loss: -0.00693\n",
      "INFO:TrainMonitor:ep: 106,\tT: 103,574,\tG: -19,\tavg_r: -0.0175,\tavg_G: -19.5,\tt: 1085,\tdt: 2.914ms,\tSimpleTD/loss: 0.0111,\tPPOClip/EntropyRegularizer/entropy: 1.72,\tPPOClip/loss: -0.00463\n",
      "INFO:TrainMonitor:ep: 107,\tT: 104,831,\tG: -18,\tavg_r: -0.0143,\tavg_G: -19.4,\tt: 1256,\tdt: 2.882ms,\tSimpleTD/loss: 0.0119,\tPPOClip/EntropyRegularizer/entropy: 1.72,\tPPOClip/loss: -0.00483\n",
      "INFO:TrainMonitor:ep: 108,\tT: 105,900,\tG: -20,\tavg_r: -0.0187,\tavg_G: -19.5,\tt: 1068,\tdt: 2.854ms,\tSimpleTD/loss: 0.0118,\tPPOClip/EntropyRegularizer/entropy: 1.71,\tPPOClip/loss: -0.00526\n",
      "INFO:TrainMonitor:ep: 109,\tT: 106,942,\tG: -20,\tavg_r: -0.0192,\tavg_G: -19.5,\tt: 1041,\tdt: 2.866ms,\tSimpleTD/loss: 0.0112,\tPPOClip/EntropyRegularizer/entropy: 1.73,\tPPOClip/loss: -0.00725\n",
      "INFO:TrainMonitor:ep: 110,\tT: 108,087,\tG: -18,\tavg_r: -0.0157,\tavg_G: -19.4,\tt: 1144,\tdt: 2.843ms,\tSimpleTD/loss: 0.0146,\tPPOClip/EntropyRegularizer/entropy: 1.72,\tPPOClip/loss: -0.00286\n",
      "INFO:TrainMonitor:ep: 111,\tT: 109,308,\tG: -19,\tavg_r: -0.0156,\tavg_G: -19.3,\tt: 1220,\tdt: 2.887ms,\tSimpleTD/loss: 0.0153,\tPPOClip/EntropyRegularizer/entropy: 1.71,\tPPOClip/loss: -0.00466\n",
      "INFO:TrainMonitor:ep: 112,\tT: 110,671,\tG: -17,\tavg_r: -0.0125,\tavg_G: -19.1,\tt: 1362,\tdt: 5.787ms,\tSimpleTD/loss: 0.0124,\tPPOClip/EntropyRegularizer/entropy: 1.71,\tPPOClip/loss: -0.00692\n",
      "INFO:TrainMonitor:ep: 113,\tT: 111,720,\tG: -19,\tavg_r: -0.0181,\tavg_G: -19.1,\tt: 1048,\tdt: 2.840ms,\tSimpleTD/loss: 0.012,\tPPOClip/EntropyRegularizer/entropy: 1.72,\tPPOClip/loss: -0.0059\n",
      "INFO:TrainMonitor:ep: 114,\tT: 112,990,\tG: -17,\tavg_r: -0.0134,\tavg_G: -18.9,\tt: 1269,\tdt: 2.862ms,\tSimpleTD/loss: 0.0128,\tPPOClip/EntropyRegularizer/entropy: 1.74,\tPPOClip/loss: -0.00362\n",
      "INFO:TrainMonitor:ep: 115,\tT: 113,830,\tG: -21,\tavg_r: -0.025,\tavg_G: -19.1,\tt: 839,\tdt: 2.935ms,\tSimpleTD/loss: 0.0108,\tPPOClip/EntropyRegularizer/entropy: 1.74,\tPPOClip/loss: -0.00411\n",
      "INFO:TrainMonitor:ep: 116,\tT: 114,963,\tG: -20,\tavg_r: -0.0177,\tavg_G: -19.2,\tt: 1132,\tdt: 2.854ms,\tSimpleTD/loss: 0.0127,\tPPOClip/EntropyRegularizer/entropy: 1.73,\tPPOClip/loss: -0.00491\n",
      "INFO:TrainMonitor:ep: 117,\tT: 115,852,\tG: -20,\tavg_r: -0.0225,\tavg_G: -19.3,\tt: 888,\tdt: 2.894ms,\tSimpleTD/loss: 0.0116,\tPPOClip/EntropyRegularizer/entropy: 1.71,\tPPOClip/loss: -0.00624\n",
      "INFO:TrainMonitor:ep: 118,\tT: 117,116,\tG: -17,\tavg_r: -0.0135,\tavg_G: -19,\tt: 1263,\tdt: 2.848ms,\tSimpleTD/loss: 0.0146,\tPPOClip/EntropyRegularizer/entropy: 1.72,\tPPOClip/loss: -0.00562\n",
      "INFO:TrainMonitor:ep: 119,\tT: 118,644,\tG: -14,\tavg_r: -0.00917,\tavg_G: -18.5,\tt: 1527,\tdt: 2.815ms,\tSimpleTD/loss: 0.0165,\tPPOClip/EntropyRegularizer/entropy: 1.72,\tPPOClip/loss: -0.00645\n",
      "INFO:TrainMonitor:ep: 120,\tT: 119,690,\tG: -20,\tavg_r: -0.0191,\tavg_G: -18.7,\tt: 1045,\tdt: 2.961ms,\tSimpleTD/loss: 0.0138,\tPPOClip/EntropyRegularizer/entropy: 1.73,\tPPOClip/loss: -0.00469\n",
      "INFO:TrainMonitor:ep: 121,\tT: 120,726,\tG: -20,\tavg_r: -0.0193,\tavg_G: -18.8,\tt: 1035,\tdt: 6.037ms,\tSimpleTD/loss: 0.0139,\tPPOClip/EntropyRegularizer/entropy: 1.73,\tPPOClip/loss: -0.0033\n",
      "INFO:TrainMonitor:ep: 122,\tT: 121,933,\tG: -19,\tavg_r: -0.0158,\tavg_G: -18.8,\tt: 1206,\tdt: 2.812ms,\tSimpleTD/loss: 0.0163,\tPPOClip/EntropyRegularizer/entropy: 1.74,\tPPOClip/loss: -0.00562\n",
      "INFO:TrainMonitor:ep: 123,\tT: 123,311,\tG: -17,\tavg_r: -0.0123,\tavg_G: -18.6,\tt: 1377,\tdt: 2.915ms,\tSimpleTD/loss: 0.0144,\tPPOClip/EntropyRegularizer/entropy: 1.73,\tPPOClip/loss: -0.0048\n",
      "INFO:TrainMonitor:ep: 124,\tT: 124,609,\tG: -16,\tavg_r: -0.0123,\tavg_G: -18.4,\tt: 1297,\tdt: 2.852ms,\tSimpleTD/loss: 0.0184,\tPPOClip/EntropyRegularizer/entropy: 1.72,\tPPOClip/loss: -0.00678\n",
      "INFO:TrainMonitor:ep: 125,\tT: 125,709,\tG: -18,\tavg_r: -0.0164,\tavg_G: -18.3,\tt: 1099,\tdt: 2.847ms,\tSimpleTD/loss: 0.0133,\tPPOClip/EntropyRegularizer/entropy: 1.72,\tPPOClip/loss: -0.00519\n",
      "INFO:TrainMonitor:ep: 126,\tT: 127,088,\tG: -19,\tavg_r: -0.0138,\tavg_G: -18.4,\tt: 1378,\tdt: 2.819ms,\tSimpleTD/loss: 0.013,\tPPOClip/EntropyRegularizer/entropy: 1.72,\tPPOClip/loss: -0.00468\n",
      "INFO:TrainMonitor:ep: 127,\tT: 128,137,\tG: -18,\tavg_r: -0.0172,\tavg_G: -18.4,\tt: 1048,\tdt: 2.961ms,\tSimpleTD/loss: 0.0129,\tPPOClip/EntropyRegularizer/entropy: 1.7,\tPPOClip/loss: -0.00722\n",
      "INFO:TrainMonitor:ep: 128,\tT: 129,186,\tG: -19,\tavg_r: -0.0181,\tavg_G: -18.4,\tt: 1048,\tdt: 2.859ms,\tSimpleTD/loss: 0.0138,\tPPOClip/EntropyRegularizer/entropy: 1.73,\tPPOClip/loss: -0.00411\n",
      "INFO:TrainMonitor:ep: 129,\tT: 130,615,\tG: -19,\tavg_r: -0.0133,\tavg_G: -18.5,\tt: 1428,\tdt: 5.104ms,\tSimpleTD/loss: 0.0162,\tPPOClip/EntropyRegularizer/entropy: 1.72,\tPPOClip/loss: -0.00672\n",
      "INFO:TrainMonitor:ep: 130,\tT: 131,811,\tG: -19,\tavg_r: -0.0159,\tavg_G: -18.5,\tt: 1195,\tdt: 2.874ms,\tSimpleTD/loss: 0.015,\tPPOClip/EntropyRegularizer/entropy: 1.7,\tPPOClip/loss: -0.00477\n",
      "INFO:TrainMonitor:ep: 131,\tT: 133,124,\tG: -17,\tavg_r: -0.013,\tavg_G: -18.4,\tt: 1312,\tdt: 2.833ms,\tSimpleTD/loss: 0.0173,\tPPOClip/EntropyRegularizer/entropy: 1.72,\tPPOClip/loss: -0.00644\n",
      "INFO:TrainMonitor:ep: 132,\tT: 134,046,\tG: -21,\tavg_r: -0.0228,\tavg_G: -18.6,\tt: 921,\tdt: 2.880ms,\tSimpleTD/loss: 0.0135,\tPPOClip/EntropyRegularizer/entropy: 1.7,\tPPOClip/loss: -0.00634\n",
      "INFO:TrainMonitor:ep: 133,\tT: 135,097,\tG: -19,\tavg_r: -0.0181,\tavg_G: -18.7,\tt: 1050,\tdt: 2.816ms,\tSimpleTD/loss: 0.0142,\tPPOClip/EntropyRegularizer/entropy: 1.71,\tPPOClip/loss: -0.00468\n",
      "INFO:TrainMonitor:ep: 134,\tT: 136,412,\tG: -16,\tavg_r: -0.0122,\tavg_G: -18.4,\tt: 1314,\tdt: 2.850ms,\tSimpleTD/loss: 0.0171,\tPPOClip/EntropyRegularizer/entropy: 1.7,\tPPOClip/loss: -0.00732\n",
      "INFO:TrainMonitor:ep: 135,\tT: 137,626,\tG: -19,\tavg_r: -0.0157,\tavg_G: -18.5,\tt: 1213,\tdt: 2.882ms,\tSimpleTD/loss: 0.0186,\tPPOClip/EntropyRegularizer/entropy: 1.7,\tPPOClip/loss: -0.00523\n",
      "INFO:TrainMonitor:ep: 136,\tT: 138,757,\tG: -18,\tavg_r: -0.0159,\tavg_G: -18.4,\tt: 1130,\tdt: 2.818ms,\tSimpleTD/loss: 0.0166,\tPPOClip/EntropyRegularizer/entropy: 1.71,\tPPOClip/loss: -0.0044\n",
      "INFO:TrainMonitor:ep: 137,\tT: 140,127,\tG: -17,\tavg_r: -0.0124,\tavg_G: -18.3,\tt: 1369,\tdt: 6.013ms,\tSimpleTD/loss: 0.0156,\tPPOClip/EntropyRegularizer/entropy: 1.69,\tPPOClip/loss: -0.00898\n",
      "INFO:TrainMonitor:ep: 138,\tT: 141,450,\tG: -18,\tavg_r: -0.0136,\tavg_G: -18.3,\tt: 1322,\tdt: 2.827ms,\tSimpleTD/loss: 0.0165,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.004\n",
      "INFO:TrainMonitor:ep: 139,\tT: 142,909,\tG: -17,\tavg_r: -0.0117,\tavg_G: -18.1,\tt: 1458,\tdt: 2.856ms,\tSimpleTD/loss: 0.0157,\tPPOClip/EntropyRegularizer/entropy: 1.7,\tPPOClip/loss: -0.00563\n",
      "INFO:TrainMonitor:ep: 140,\tT: 144,427,\tG: -15,\tavg_r: -0.00989,\tavg_G: -17.8,\tt: 1517,\tdt: 2.859ms,\tSimpleTD/loss: 0.0172,\tPPOClip/EntropyRegularizer/entropy: 1.69,\tPPOClip/loss: -0.00573\n",
      "INFO:TrainMonitor:ep: 141,\tT: 146,213,\tG: -14,\tavg_r: -0.00784,\tavg_G: -17.4,\tt: 1785,\tdt: 2.831ms,\tSimpleTD/loss: 0.0161,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.00686\n",
      "INFO:TrainMonitor:ep: 142,\tT: 147,115,\tG: -21,\tavg_r: -0.0233,\tavg_G: -17.8,\tt: 901,\tdt: 2.869ms,\tSimpleTD/loss: 0.0139,\tPPOClip/EntropyRegularizer/entropy: 1.7,\tPPOClip/loss: -0.00269\n",
      "INFO:TrainMonitor:ep: 143,\tT: 148,191,\tG: -21,\tavg_r: -0.0195,\tavg_G: -18.1,\tt: 1075,\tdt: 2.820ms,\tSimpleTD/loss: 0.0151,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.00618\n",
      "INFO:TrainMonitor:ep: 144,\tT: 149,451,\tG: -18,\tavg_r: -0.0143,\tavg_G: -18.1,\tt: 1259,\tdt: 2.853ms,\tSimpleTD/loss: 0.0146,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00505\n",
      "INFO:TrainMonitor:ep: 145,\tT: 150,841,\tG: -16,\tavg_r: -0.0115,\tavg_G: -17.9,\tt: 1389,\tdt: 6.967ms,\tSimpleTD/loss: 0.0187,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.00677\n",
      "INFO:TrainMonitor:ep: 146,\tT: 152,004,\tG: -21,\tavg_r: -0.0181,\tavg_G: -18.2,\tt: 1162,\tdt: 2.880ms,\tSimpleTD/loss: 0.017,\tPPOClip/EntropyRegularizer/entropy: 1.69,\tPPOClip/loss: -0.00724\n",
      "INFO:TrainMonitor:ep: 147,\tT: 153,246,\tG: -18,\tavg_r: -0.0145,\tavg_G: -18.2,\tt: 1241,\tdt: 2.846ms,\tSimpleTD/loss: 0.0142,\tPPOClip/EntropyRegularizer/entropy: 1.69,\tPPOClip/loss: -0.0032\n",
      "INFO:TrainMonitor:ep: 148,\tT: 154,456,\tG: -19,\tavg_r: -0.0157,\tavg_G: -18.3,\tt: 1209,\tdt: 2.801ms,\tSimpleTD/loss: 0.017,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.00534\n",
      "INFO:TrainMonitor:ep: 149,\tT: 155,547,\tG: -19,\tavg_r: -0.0174,\tavg_G: -18.3,\tt: 1090,\tdt: 2.805ms,\tSimpleTD/loss: 0.0144,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.00798\n",
      "INFO:TrainMonitor:ep: 150,\tT: 157,133,\tG: -14,\tavg_r: -0.00883,\tavg_G: -17.9,\tt: 1585,\tdt: 2.879ms,\tSimpleTD/loss: 0.0162,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00568\n",
      "INFO:TrainMonitor:ep: 151,\tT: 158,607,\tG: -18,\tavg_r: -0.0122,\tavg_G: -17.9,\tt: 1473,\tdt: 2.802ms,\tSimpleTD/loss: 0.0176,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00302\n",
      "INFO:TrainMonitor:ep: 152,\tT: 160,355,\tG: -16,\tavg_r: -0.00916,\tavg_G: -17.7,\tt: 1747,\tdt: 5.410ms,\tSimpleTD/loss: 0.016,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00699\n",
      "INFO:TrainMonitor:ep: 153,\tT: 161,737,\tG: -18,\tavg_r: -0.013,\tavg_G: -17.7,\tt: 1381,\tdt: 2.864ms,\tSimpleTD/loss: 0.0166,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00516\n",
      "INFO:TrainMonitor:ep: 154,\tT: 163,183,\tG: -17,\tavg_r: -0.0118,\tavg_G: -17.7,\tt: 1445,\tdt: 2.786ms,\tSimpleTD/loss: 0.0147,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00609\n",
      "INFO:TrainMonitor:ep: 155,\tT: 164,683,\tG: -16,\tavg_r: -0.0107,\tavg_G: -17.5,\tt: 1499,\tdt: 2.829ms,\tSimpleTD/loss: 0.0149,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.00248\n",
      "INFO:TrainMonitor:ep: 156,\tT: 166,345,\tG: -16,\tavg_r: -0.00963,\tavg_G: -17.4,\tt: 1661,\tdt: 2.869ms,\tSimpleTD/loss: 0.0176,\tPPOClip/EntropyRegularizer/entropy: 1.69,\tPPOClip/loss: -0.00654\n",
      "INFO:TrainMonitor:ep: 157,\tT: 167,642,\tG: -18,\tavg_r: -0.0139,\tavg_G: -17.4,\tt: 1296,\tdt: 2.844ms,\tSimpleTD/loss: 0.0182,\tPPOClip/EntropyRegularizer/entropy: 1.69,\tPPOClip/loss: -0.0029\n",
      "INFO:TrainMonitor:ep: 158,\tT: 169,435,\tG: -12,\tavg_r: -0.0067,\tavg_G: -16.9,\tt: 1792,\tdt: 2.873ms,\tSimpleTD/loss: 0.0174,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00814\n",
      "INFO:TrainMonitor:ep: 159,\tT: 171,147,\tG: -15,\tavg_r: -0.00877,\tavg_G: -16.7,\tt: 1711,\tdt: 5.442ms,\tSimpleTD/loss: 0.018,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00568\n",
      "INFO:TrainMonitor:ep: 160,\tT: 172,513,\tG: -17,\tavg_r: -0.0125,\tavg_G: -16.7,\tt: 1365,\tdt: 2.867ms,\tSimpleTD/loss: 0.0202,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00597\n",
      "INFO:TrainMonitor:ep: 161,\tT: 173,903,\tG: -18,\tavg_r: -0.013,\tavg_G: -16.8,\tt: 1389,\tdt: 2.792ms,\tSimpleTD/loss: 0.0162,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.00668\n",
      "INFO:TrainMonitor:ep: 162,\tT: 175,103,\tG: -17,\tavg_r: -0.0142,\tavg_G: -16.9,\tt: 1199,\tdt: 2.850ms,\tSimpleTD/loss: 0.0197,\tPPOClip/EntropyRegularizer/entropy: 1.69,\tPPOClip/loss: -0.00465\n",
      "INFO:TrainMonitor:ep: 163,\tT: 176,937,\tG: -14,\tavg_r: -0.00764,\tavg_G: -16.6,\tt: 1833,\tdt: 2.812ms,\tSimpleTD/loss: 0.0168,\tPPOClip/EntropyRegularizer/entropy: 1.7,\tPPOClip/loss: -0.00549\n",
      "INFO:TrainMonitor:ep: 164,\tT: 178,398,\tG: -16,\tavg_r: -0.011,\tavg_G: -16.5,\tt: 1460,\tdt: 2.834ms,\tSimpleTD/loss: 0.0174,\tPPOClip/EntropyRegularizer/entropy: 1.69,\tPPOClip/loss: -0.00598\n",
      "INFO:TrainMonitor:ep: 165,\tT: 179,639,\tG: -19,\tavg_r: -0.0153,\tavg_G: -16.8,\tt: 1240,\tdt: 2.824ms,\tSimpleTD/loss: 0.0169,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.00767\n",
      "INFO:TrainMonitor:ep: 166,\tT: 181,163,\tG: -17,\tavg_r: -0.0112,\tavg_G: -16.8,\tt: 1523,\tdt: 6.195ms,\tSimpleTD/loss: 0.0169,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00579\n",
      "INFO:TrainMonitor:ep: 167,\tT: 182,648,\tG: -18,\tavg_r: -0.0121,\tavg_G: -16.9,\tt: 1484,\tdt: 2.851ms,\tSimpleTD/loss: 0.0151,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.00741\n",
      "INFO:TrainMonitor:ep: 168,\tT: 184,234,\tG: -15,\tavg_r: -0.00946,\tavg_G: -16.7,\tt: 1585,\tdt: 2.825ms,\tSimpleTD/loss: 0.0159,\tPPOClip/EntropyRegularizer/entropy: 1.69,\tPPOClip/loss: -0.00671\n",
      "INFO:TrainMonitor:ep: 169,\tT: 185,823,\tG: -14,\tavg_r: -0.00882,\tavg_G: -16.4,\tt: 1588,\tdt: 2.884ms,\tSimpleTD/loss: 0.0188,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.0065\n",
      "INFO:TrainMonitor:ep: 170,\tT: 187,129,\tG: -18,\tavg_r: -0.0138,\tavg_G: -16.6,\tt: 1305,\tdt: 2.820ms,\tSimpleTD/loss: 0.0165,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.00614\n",
      "INFO:TrainMonitor:ep: 171,\tT: 188,334,\tG: -18,\tavg_r: -0.015,\tavg_G: -16.7,\tt: 1204,\tdt: 2.816ms,\tSimpleTD/loss: 0.0138,\tPPOClip/EntropyRegularizer/entropy: 1.69,\tPPOClip/loss: -0.00436\n",
      "INFO:TrainMonitor:ep: 172,\tT: 189,549,\tG: -19,\tavg_r: -0.0157,\tavg_G: -17,\tt: 1214,\tdt: 2.849ms,\tSimpleTD/loss: 0.0173,\tPPOClip/EntropyRegularizer/entropy: 1.69,\tPPOClip/loss: -0.00518\n",
      "INFO:TrainMonitor:ep: 173,\tT: 190,947,\tG: -17,\tavg_r: -0.0122,\tavg_G: -17,\tt: 1397,\tdt: 5.680ms,\tSimpleTD/loss: 0.0179,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00763\n",
      "INFO:TrainMonitor:ep: 174,\tT: 192,521,\tG: -15,\tavg_r: -0.00954,\tavg_G: -16.8,\tt: 1573,\tdt: 2.814ms,\tSimpleTD/loss: 0.0162,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00439\n",
      "INFO:TrainMonitor:ep: 175,\tT: 194,051,\tG: -18,\tavg_r: -0.0118,\tavg_G: -16.9,\tt: 1529,\tdt: 2.817ms,\tSimpleTD/loss: 0.0164,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00694\n",
      "INFO:TrainMonitor:ep: 176,\tT: 195,693,\tG: -15,\tavg_r: -0.00914,\tavg_G: -16.7,\tt: 1641,\tdt: 2.823ms,\tSimpleTD/loss: 0.0151,\tPPOClip/EntropyRegularizer/entropy: 1.69,\tPPOClip/loss: -0.00687\n",
      "INFO:TrainMonitor:ep: 177,\tT: 197,013,\tG: -17,\tavg_r: -0.0129,\tavg_G: -16.7,\tt: 1319,\tdt: 2.804ms,\tSimpleTD/loss: 0.014,\tPPOClip/EntropyRegularizer/entropy: 1.7,\tPPOClip/loss: -0.00503\n",
      "INFO:TrainMonitor:ep: 178,\tT: 198,429,\tG: -16,\tavg_r: -0.0113,\tavg_G: -16.7,\tt: 1415,\tdt: 2.845ms,\tSimpleTD/loss: 0.0198,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.00687\n",
      "INFO:TrainMonitor:ep: 179,\tT: 199,636,\tG: -19,\tavg_r: -0.0158,\tavg_G: -16.9,\tt: 1206,\tdt: 2.856ms,\tSimpleTD/loss: 0.0147,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00484\n",
      "INFO:TrainMonitor:ep: 180,\tT: 201,104,\tG: -16,\tavg_r: -0.0109,\tavg_G: -16.8,\tt: 1467,\tdt: 5.642ms,\tSimpleTD/loss: 0.0151,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.00586\n",
      "INFO:TrainMonitor:ep: 181,\tT: 202,643,\tG: -17,\tavg_r: -0.0111,\tavg_G: -16.8,\tt: 1538,\tdt: 2.863ms,\tSimpleTD/loss: 0.0164,\tPPOClip/EntropyRegularizer/entropy: 1.7,\tPPOClip/loss: -0.00509\n",
      "INFO:TrainMonitor:ep: 182,\tT: 204,217,\tG: -15,\tavg_r: -0.00954,\tavg_G: -16.6,\tt: 1573,\tdt: 2.840ms,\tSimpleTD/loss: 0.0149,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.00639\n",
      "INFO:TrainMonitor:ep: 183,\tT: 205,353,\tG: -18,\tavg_r: -0.0159,\tavg_G: -16.8,\tt: 1135,\tdt: 2.853ms,\tSimpleTD/loss: 0.0144,\tPPOClip/EntropyRegularizer/entropy: 1.69,\tPPOClip/loss: -0.00347\n",
      "INFO:TrainMonitor:ep: 184,\tT: 206,720,\tG: -16,\tavg_r: -0.0117,\tavg_G: -16.7,\tt: 1366,\tdt: 2.806ms,\tSimpleTD/loss: 0.014,\tPPOClip/EntropyRegularizer/entropy: 1.69,\tPPOClip/loss: -0.00567\n",
      "INFO:TrainMonitor:ep: 185,\tT: 208,094,\tG: -18,\tavg_r: -0.0131,\tavg_G: -16.8,\tt: 1373,\tdt: 2.857ms,\tSimpleTD/loss: 0.0157,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.00618\n",
      "INFO:TrainMonitor:ep: 186,\tT: 209,826,\tG: -15,\tavg_r: -0.00867,\tavg_G: -16.6,\tt: 1731,\tdt: 2.785ms,\tSimpleTD/loss: 0.0148,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00568\n",
      "INFO:TrainMonitor:ep: 187,\tT: 211,238,\tG: -18,\tavg_r: -0.0128,\tavg_G: -16.8,\tt: 1411,\tdt: 6.745ms,\tSimpleTD/loss: 0.0125,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00735\n",
      "INFO:TrainMonitor:ep: 188,\tT: 212,996,\tG: -15,\tavg_r: -0.00854,\tavg_G: -16.6,\tt: 1757,\tdt: 2.821ms,\tSimpleTD/loss: 0.0157,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00512\n",
      "INFO:TrainMonitor:ep: 189,\tT: 214,840,\tG: -14,\tavg_r: -0.0076,\tavg_G: -16.3,\tt: 1843,\tdt: 2.836ms,\tSimpleTD/loss: 0.0131,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00493\n",
      "INFO:TrainMonitor:ep: 190,\tT: 216,156,\tG: -17,\tavg_r: -0.0129,\tavg_G: -16.4,\tt: 1315,\tdt: 2.859ms,\tSimpleTD/loss: 0.0197,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.00616\n",
      "INFO:TrainMonitor:ep: 191,\tT: 218,037,\tG: -13,\tavg_r: -0.00691,\tavg_G: -16.1,\tt: 1880,\tdt: 2.827ms,\tSimpleTD/loss: 0.0163,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00581\n",
      "INFO:TrainMonitor:ep: 192,\tT: 219,582,\tG: -18,\tavg_r: -0.0117,\tavg_G: -16.3,\tt: 1544,\tdt: 2.865ms,\tSimpleTD/loss: 0.0153,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00569\n",
      "INFO:TrainMonitor:ep: 193,\tT: 221,286,\tG: -17,\tavg_r: -0.00998,\tavg_G: -16.3,\tt: 1703,\tdt: 6.198ms,\tSimpleTD/loss: 0.0177,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00543\n",
      "INFO:TrainMonitor:ep: 194,\tT: 222,950,\tG: -15,\tavg_r: -0.00902,\tavg_G: -16.2,\tt: 1663,\tdt: 2.864ms,\tSimpleTD/loss: 0.0148,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00641\n",
      "INFO:TrainMonitor:ep: 195,\tT: 224,791,\tG: -15,\tavg_r: -0.00815,\tavg_G: -16.1,\tt: 1840,\tdt: 2.842ms,\tSimpleTD/loss: 0.0147,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.0057\n",
      "INFO:TrainMonitor:ep: 196,\tT: 226,657,\tG: -16,\tavg_r: -0.00858,\tavg_G: -16.1,\tt: 1865,\tdt: 2.824ms,\tSimpleTD/loss: 0.0138,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00534\n",
      "INFO:TrainMonitor:ep: 197,\tT: 228,418,\tG: -13,\tavg_r: -0.00739,\tavg_G: -15.8,\tt: 1760,\tdt: 2.849ms,\tSimpleTD/loss: 0.0178,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.00593\n",
      "INFO:TrainMonitor:ep: 198,\tT: 230,503,\tG: -11,\tavg_r: -0.00528,\tavg_G: -15.3,\tt: 2084,\tdt: 5.348ms,\tSimpleTD/loss: 0.0177,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00441\n",
      "INFO:TrainMonitor:ep: 199,\tT: 231,896,\tG: -18,\tavg_r: -0.0129,\tavg_G: -15.6,\tt: 1392,\tdt: 2.898ms,\tSimpleTD/loss: 0.0148,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00585\n",
      "INFO:TrainMonitor:ep: 200,\tT: 233,607,\tG: -15,\tavg_r: -0.00877,\tavg_G: -15.5,\tt: 1710,\tdt: 2.788ms,\tSimpleTD/loss: 0.0172,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00451\n",
      "INFO:TrainMonitor:ep: 201,\tT: 235,080,\tG: -17,\tavg_r: -0.0115,\tavg_G: -15.7,\tt: 1472,\tdt: 2.831ms,\tSimpleTD/loss: 0.0164,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00613\n",
      "INFO:TrainMonitor:ep: 202,\tT: 236,689,\tG: -15,\tavg_r: -0.00933,\tavg_G: -15.6,\tt: 1608,\tdt: 2.804ms,\tSimpleTD/loss: 0.0159,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00384\n",
      "INFO:TrainMonitor:ep: 203,\tT: 238,197,\tG: -17,\tavg_r: -0.0113,\tavg_G: -15.7,\tt: 1507,\tdt: 2.823ms,\tSimpleTD/loss: 0.0181,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00755\n",
      "INFO:TrainMonitor:ep: 204,\tT: 239,786,\tG: -17,\tavg_r: -0.0107,\tavg_G: -15.9,\tt: 1588,\tdt: 2.808ms,\tSimpleTD/loss: 0.0161,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00577\n",
      "INFO:TrainMonitor:ep: 205,\tT: 241,145,\tG: -17,\tavg_r: -0.0125,\tavg_G: -16,\tt: 1358,\tdt: 6.927ms,\tSimpleTD/loss: 0.0159,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00562\n",
      "INFO:TrainMonitor:ep: 206,\tT: 242,598,\tG: -17,\tavg_r: -0.0117,\tavg_G: -16.1,\tt: 1452,\tdt: 2.806ms,\tSimpleTD/loss: 0.0122,\tPPOClip/EntropyRegularizer/entropy: 1.64,\tPPOClip/loss: -0.00398\n",
      "INFO:TrainMonitor:ep: 207,\tT: 244,205,\tG: -16,\tavg_r: -0.00996,\tavg_G: -16.1,\tt: 1606,\tdt: 2.848ms,\tSimpleTD/loss: 0.0155,\tPPOClip/EntropyRegularizer/entropy: 1.64,\tPPOClip/loss: -0.00565\n",
      "INFO:TrainMonitor:ep: 208,\tT: 246,131,\tG: -14,\tavg_r: -0.00727,\tavg_G: -15.9,\tt: 1925,\tdt: 2.780ms,\tSimpleTD/loss: 0.0141,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00562\n",
      "INFO:TrainMonitor:ep: 209,\tT: 248,026,\tG: -11,\tavg_r: -0.00581,\tavg_G: -15.4,\tt: 1894,\tdt: 2.780ms,\tSimpleTD/loss: 0.0133,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00429\n",
      "INFO:TrainMonitor:ep: 210,\tT: 250,071,\tG: -11,\tavg_r: -0.00538,\tavg_G: -14.9,\tt: 2044,\tdt: 5.675ms,\tSimpleTD/loss: 0.0118,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00585\n",
      "INFO:TrainMonitor:ep: 211,\tT: 251,670,\tG: -15,\tavg_r: -0.00939,\tavg_G: -14.9,\tt: 1598,\tdt: 2.846ms,\tSimpleTD/loss: 0.0148,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00411\n",
      "INFO:TrainMonitor:ep: 212,\tT: 253,584,\tG: -13,\tavg_r: -0.0068,\tavg_G: -14.7,\tt: 1913,\tdt: 2.794ms,\tSimpleTD/loss: 0.016,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00519\n",
      "INFO:TrainMonitor:ep: 213,\tT: 255,337,\tG: -15,\tavg_r: -0.00856,\tavg_G: -14.8,\tt: 1752,\tdt: 2.803ms,\tSimpleTD/loss: 0.015,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00623\n",
      "INFO:TrainMonitor:ep: 214,\tT: 256,544,\tG: -19,\tavg_r: -0.0158,\tavg_G: -15.2,\tt: 1206,\tdt: 2.848ms,\tSimpleTD/loss: 0.0172,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00566\n",
      "INFO:TrainMonitor:ep: 215,\tT: 258,575,\tG: -10,\tavg_r: -0.00493,\tavg_G: -14.7,\tt: 2030,\tdt: 2.819ms,\tSimpleTD/loss: 0.0147,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00369\n",
      "INFO:TrainMonitor:ep: 216,\tT: 260,403,\tG: -16,\tavg_r: -0.00876,\tavg_G: -14.8,\tt: 1827,\tdt: 5.538ms,\tSimpleTD/loss: 0.0125,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00672\n",
      "INFO:TrainMonitor:ep: 217,\tT: 262,259,\tG: -11,\tavg_r: -0.00593,\tavg_G: -14.4,\tt: 1855,\tdt: 2.804ms,\tSimpleTD/loss: 0.018,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00461\n",
      "INFO:TrainMonitor:ep: 218,\tT: 264,580,\tG: -7,\tavg_r: -0.00302,\tavg_G: -13.7,\tt: 2320,\tdt: 2.805ms,\tSimpleTD/loss: 0.0134,\tPPOClip/EntropyRegularizer/entropy: 1.69,\tPPOClip/loss: -0.00426\n",
      "INFO:TrainMonitor:ep: 219,\tT: 266,303,\tG: -14,\tavg_r: -0.00813,\tavg_G: -13.7,\tt: 1722,\tdt: 2.823ms,\tSimpleTD/loss: 0.0197,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.00672\n",
      "INFO:TrainMonitor:ep: 220,\tT: 267,866,\tG: -17,\tavg_r: -0.0109,\tavg_G: -14,\tt: 1562,\tdt: 2.788ms,\tSimpleTD/loss: 0.0162,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00457\n",
      "INFO:TrainMonitor:ep: 221,\tT: 269,773,\tG: -12,\tavg_r: -0.0063,\tavg_G: -13.8,\tt: 1906,\tdt: 2.806ms,\tSimpleTD/loss: 0.0146,\tPPOClip/EntropyRegularizer/entropy: 1.68,\tPPOClip/loss: -0.00529\n",
      "INFO:TrainMonitor:ep: 222,\tT: 271,485,\tG: -14,\tavg_r: -0.00818,\tavg_G: -13.9,\tt: 1711,\tdt: 5.229ms,\tSimpleTD/loss: 0.0179,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00706\n",
      "INFO:TrainMonitor:ep: 223,\tT: 273,399,\tG: -12,\tavg_r: -0.00627,\tavg_G: -13.7,\tt: 1913,\tdt: 2.856ms,\tSimpleTD/loss: 0.0154,\tPPOClip/EntropyRegularizer/entropy: 1.69,\tPPOClip/loss: -0.00572\n",
      "INFO:TrainMonitor:ep: 224,\tT: 275,062,\tG: -16,\tavg_r: -0.00963,\tavg_G: -13.9,\tt: 1662,\tdt: 2.787ms,\tSimpleTD/loss: 0.0178,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00684\n",
      "INFO:TrainMonitor:ep: 225,\tT: 276,879,\tG: -14,\tavg_r: -0.00771,\tavg_G: -13.9,\tt: 1816,\tdt: 2.794ms,\tSimpleTD/loss: 0.0128,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.0058\n",
      "INFO:TrainMonitor:ep: 226,\tT: 278,843,\tG: -12,\tavg_r: -0.00611,\tavg_G: -13.7,\tt: 1963,\tdt: 2.828ms,\tSimpleTD/loss: 0.014,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00647\n",
      "INFO:TrainMonitor:ep: 227,\tT: 280,825,\tG: -11,\tavg_r: -0.00555,\tavg_G: -13.4,\tt: 1981,\tdt: 5.191ms,\tSimpleTD/loss: 0.0129,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00598\n",
      "INFO:TrainMonitor:ep: 228,\tT: 282,904,\tG: -10,\tavg_r: -0.00481,\tavg_G: -13.1,\tt: 2078,\tdt: 2.863ms,\tSimpleTD/loss: 0.0161,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00373\n",
      "INFO:TrainMonitor:ep: 229,\tT: 285,234,\tG: -13,\tavg_r: -0.00558,\tavg_G: -13.1,\tt: 2329,\tdt: 2.796ms,\tSimpleTD/loss: 0.0147,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00644\n",
      "INFO:TrainMonitor:ep: 230,\tT: 286,733,\tG: -14,\tavg_r: -0.00935,\tavg_G: -13.2,\tt: 1498,\tdt: 2.790ms,\tSimpleTD/loss: 0.0185,\tPPOClip/EntropyRegularizer/entropy: 1.64,\tPPOClip/loss: -0.00582\n",
      "INFO:TrainMonitor:ep: 231,\tT: 288,452,\tG: -15,\tavg_r: -0.00873,\tavg_G: -13.4,\tt: 1718,\tdt: 2.767ms,\tSimpleTD/loss: 0.0176,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.0046\n",
      "INFO:TrainMonitor:ep: 232,\tT: 290,311,\tG: -13,\tavg_r: -0.007,\tavg_G: -13.3,\tt: 1858,\tdt: 6.601ms,\tSimpleTD/loss: 0.017,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00642\n",
      "INFO:TrainMonitor:ep: 233,\tT: 291,818,\tG: -14,\tavg_r: -0.0093,\tavg_G: -13.4,\tt: 1506,\tdt: 2.759ms,\tSimpleTD/loss: 0.0174,\tPPOClip/EntropyRegularizer/entropy: 1.64,\tPPOClip/loss: -0.00756\n",
      "INFO:TrainMonitor:ep: 234,\tT: 293,844,\tG: -13,\tavg_r: -0.00642,\tavg_G: -13.4,\tt: 2025,\tdt: 2.808ms,\tSimpleTD/loss: 0.0178,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00614\n",
      "INFO:TrainMonitor:ep: 235,\tT: 296,047,\tG: -9,\tavg_r: -0.00409,\tavg_G: -12.9,\tt: 2202,\tdt: 2.807ms,\tSimpleTD/loss: 0.0161,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00567\n",
      "INFO:TrainMonitor:ep: 236,\tT: 297,829,\tG: -14,\tavg_r: -0.00786,\tavg_G: -13,\tt: 1781,\tdt: 2.815ms,\tSimpleTD/loss: 0.0183,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00588\n",
      "INFO:TrainMonitor:ep: 237,\tT: 299,915,\tG: -13,\tavg_r: -0.00624,\tavg_G: -13,\tt: 2085,\tdt: 2.814ms,\tSimpleTD/loss: 0.0144,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00504\n",
      "INFO:TrainMonitor:ep: 238,\tT: 301,803,\tG: -13,\tavg_r: -0.00689,\tavg_G: -13,\tt: 1887,\tdt: 5.526ms,\tSimpleTD/loss: 0.0167,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00648\n",
      "INFO:TrainMonitor:ep: 239,\tT: 303,731,\tG: -13,\tavg_r: -0.00675,\tavg_G: -13,\tt: 1927,\tdt: 2.835ms,\tSimpleTD/loss: 0.0174,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00464\n",
      "INFO:TrainMonitor:ep: 240,\tT: 305,725,\tG: -12,\tavg_r: -0.00602,\tavg_G: -12.9,\tt: 1993,\tdt: 2.845ms,\tSimpleTD/loss: 0.0128,\tPPOClip/EntropyRegularizer/entropy: 1.64,\tPPOClip/loss: -0.00513\n",
      "INFO:TrainMonitor:ep: 241,\tT: 307,857,\tG: -11,\tavg_r: -0.00516,\tavg_G: -12.7,\tt: 2131,\tdt: 2.815ms,\tSimpleTD/loss: 0.017,\tPPOClip/EntropyRegularizer/entropy: 1.64,\tPPOClip/loss: -0.00516\n",
      "INFO:TrainMonitor:ep: 242,\tT: 309,682,\tG: -15,\tavg_r: -0.00822,\tavg_G: -13,\tt: 1824,\tdt: 2.800ms,\tSimpleTD/loss: 0.0155,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00465\n",
      "INFO:TrainMonitor:ep: 243,\tT: 311,746,\tG: -10,\tavg_r: -0.00485,\tavg_G: -12.7,\tt: 2063,\tdt: 5.825ms,\tSimpleTD/loss: 0.0145,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00305\n",
      "INFO:TrainMonitor:ep: 244,\tT: 313,863,\tG: -11,\tavg_r: -0.0052,\tavg_G: -12.5,\tt: 2116,\tdt: 2.843ms,\tSimpleTD/loss: 0.0173,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00517\n",
      "INFO:TrainMonitor:ep: 245,\tT: 316,153,\tG: -11,\tavg_r: -0.00481,\tavg_G: -12.3,\tt: 2289,\tdt: 2.803ms,\tSimpleTD/loss: 0.0168,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00309\n",
      "INFO:TrainMonitor:ep: 246,\tT: 317,795,\tG: -17,\tavg_r: -0.0104,\tavg_G: -12.8,\tt: 1641,\tdt: 2.851ms,\tSimpleTD/loss: 0.0139,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00304\n",
      "INFO:TrainMonitor:ep: 247,\tT: 320,184,\tG: -9,\tavg_r: -0.00377,\tavg_G: -12.4,\tt: 2388,\tdt: 5.805ms,\tSimpleTD/loss: 0.0172,\tPPOClip/EntropyRegularizer/entropy: 1.67,\tPPOClip/loss: -0.00497\n",
      "INFO:TrainMonitor:ep: 248,\tT: 322,046,\tG: -12,\tavg_r: -0.00645,\tavg_G: -12.4,\tt: 1861,\tdt: 2.810ms,\tSimpleTD/loss: 0.0159,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00451\n",
      "INFO:TrainMonitor:ep: 249,\tT: 324,109,\tG: -11,\tavg_r: -0.00533,\tavg_G: -12.2,\tt: 2062,\tdt: 2.834ms,\tSimpleTD/loss: 0.0187,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00741\n",
      "INFO:TrainMonitor:ep: 250,\tT: 326,010,\tG: -12,\tavg_r: -0.00632,\tavg_G: -12.2,\tt: 1900,\tdt: 2.788ms,\tSimpleTD/loss: 0.0187,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00544\n",
      "INFO:TrainMonitor:ep: 251,\tT: 328,061,\tG: -12,\tavg_r: -0.00585,\tavg_G: -12.2,\tt: 2050,\tdt: 2.806ms,\tSimpleTD/loss: 0.0132,\tPPOClip/EntropyRegularizer/entropy: 1.66,\tPPOClip/loss: -0.00502\n",
      "INFO:TrainMonitor:ep: 252,\tT: 330,179,\tG: -8,\tavg_r: -0.00378,\tavg_G: -11.8,\tt: 2117,\tdt: 6.170ms,\tSimpleTD/loss: 0.0186,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.0047\n",
      "INFO:TrainMonitor:ep: 253,\tT: 332,646,\tG: -7,\tavg_r: -0.00284,\tavg_G: -11.3,\tt: 2466,\tdt: 2.809ms,\tSimpleTD/loss: 0.0111,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00585\n",
      "INFO:TrainMonitor:ep: 254,\tT: 334,852,\tG: -9,\tavg_r: -0.00408,\tavg_G: -11.1,\tt: 2205,\tdt: 2.775ms,\tSimpleTD/loss: 0.0188,\tPPOClip/EntropyRegularizer/entropy: 1.64,\tPPOClip/loss: -0.00552\n",
      "INFO:TrainMonitor:ep: 255,\tT: 336,864,\tG: -10,\tavg_r: -0.00497,\tavg_G: -11,\tt: 2011,\tdt: 2.801ms,\tSimpleTD/loss: 0.0168,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00569\n",
      "INFO:TrainMonitor:ep: 256,\tT: 339,404,\tG: -6,\tavg_r: -0.00236,\tavg_G: -10.5,\tt: 2539,\tdt: 2.814ms,\tSimpleTD/loss: 0.0189,\tPPOClip/EntropyRegularizer/entropy: 1.64,\tPPOClip/loss: -0.00593\n",
      "INFO:TrainMonitor:ep: 257,\tT: 341,524,\tG: -10,\tavg_r: -0.00472,\tavg_G: -10.4,\tt: 2119,\tdt: 6.185ms,\tSimpleTD/loss: 0.0153,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00536\n",
      "INFO:TrainMonitor:ep: 258,\tT: 344,172,\tG: -5,\tavg_r: -0.00189,\tavg_G: -9.88,\tt: 2647,\tdt: 2.793ms,\tSimpleTD/loss: 0.0158,\tPPOClip/EntropyRegularizer/entropy: 1.64,\tPPOClip/loss: -0.00464\n",
      "INFO:TrainMonitor:ep: 259,\tT: 346,736,\tG: -5,\tavg_r: -0.00195,\tavg_G: -9.39,\tt: 2563,\tdt: 2.806ms,\tSimpleTD/loss: 0.014,\tPPOClip/EntropyRegularizer/entropy: 1.64,\tPPOClip/loss: -0.00557\n",
      "INFO:TrainMonitor:ep: 260,\tT: 349,199,\tG: -8,\tavg_r: -0.00325,\tavg_G: -9.25,\tt: 2462,\tdt: 2.834ms,\tSimpleTD/loss: 0.0166,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00741\n",
      "INFO:TrainMonitor:ep: 261,\tT: 351,779,\tG: -7,\tavg_r: -0.00271,\tavg_G: -9.03,\tt: 2579,\tdt: 6.052ms,\tSimpleTD/loss: 0.0174,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00688\n",
      "INFO:TrainMonitor:ep: 262,\tT: 354,201,\tG: -6,\tavg_r: -0.00248,\tavg_G: -8.72,\tt: 2421,\tdt: 2.792ms,\tSimpleTD/loss: 0.0143,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00489\n",
      "INFO:TrainMonitor:ep: 263,\tT: 356,893,\tG: -3,\tavg_r: -0.00111,\tavg_G: -8.15,\tt: 2691,\tdt: 2.802ms,\tSimpleTD/loss: 0.0167,\tPPOClip/EntropyRegularizer/entropy: 1.64,\tPPOClip/loss: -0.00711\n",
      "INFO:TrainMonitor:ep: 264,\tT: 359,607,\tG: -1,\tavg_r: -0.000369,\tavg_G: -7.44,\tt: 2713,\tdt: 2.783ms,\tSimpleTD/loss: 0.0156,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00389\n",
      "INFO:TrainMonitor:ep: 265,\tT: 361,699,\tG: -9,\tavg_r: -0.0043,\tavg_G: -7.59,\tt: 2091,\tdt: 6.551ms,\tSimpleTD/loss: 0.0202,\tPPOClip/EntropyRegularizer/entropy: 1.64,\tPPOClip/loss: -0.005\n",
      "INFO:TrainMonitor:ep: 266,\tT: 364,518,\tG: -8,\tavg_r: -0.00284,\tavg_G: -7.63,\tt: 2818,\tdt: 2.804ms,\tSimpleTD/loss: 0.0191,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00919\n",
      "INFO:TrainMonitor:ep: 267,\tT: 366,782,\tG: -10,\tavg_r: -0.00442,\tavg_G: -7.87,\tt: 2263,\tdt: 2.799ms,\tSimpleTD/loss: 0.016,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00358\n",
      "INFO:TrainMonitor:ep: 268,\tT: 368,918,\tG: -10,\tavg_r: -0.00468,\tavg_G: -8.08,\tt: 2135,\tdt: 2.776ms,\tSimpleTD/loss: 0.0227,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00722\n",
      "INFO:TrainMonitor:ep: 269,\tT: 371,426,\tG: -8,\tavg_r: -0.00319,\tavg_G: -8.07,\tt: 2507,\tdt: 5.946ms,\tSimpleTD/loss: 0.0182,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.006\n",
      "INFO:TrainMonitor:ep: 270,\tT: 374,029,\tG: -7,\tavg_r: -0.00269,\tavg_G: -7.97,\tt: 2602,\tdt: 2.795ms,\tSimpleTD/loss: 0.0104,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00408\n",
      "INFO:TrainMonitor:ep: 271,\tT: 376,698,\tG: -4,\tavg_r: -0.0015,\tavg_G: -7.57,\tt: 2668,\tdt: 2.817ms,\tSimpleTD/loss: 0.0114,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00503\n",
      "INFO:TrainMonitor:ep: 272,\tT: 379,505,\tG: -3,\tavg_r: -0.00107,\tavg_G: -7.11,\tt: 2806,\tdt: 2.804ms,\tSimpleTD/loss: 0.00995,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00397\n",
      "INFO:TrainMonitor:ep: 273,\tT: 382,578,\tG: -2,\tavg_r: -0.000651,\tavg_G: -6.6,\tt: 3072,\tdt: 5.454ms,\tSimpleTD/loss: 0.0173,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00623\n",
      "INFO:TrainMonitor:ep: 274,\tT: 385,243,\tG: -3,\tavg_r: -0.00113,\tavg_G: -6.24,\tt: 2664,\tdt: 2.798ms,\tSimpleTD/loss: 0.0138,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00526\n",
      "INFO:TrainMonitor:ep: 275,\tT: 387,883,\tG: -6,\tavg_r: -0.00227,\tavg_G: -6.22,\tt: 2639,\tdt: 2.820ms,\tSimpleTD/loss: 0.0144,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00566\n",
      "INFO:TrainMonitor:ep: 276,\tT: 390,565,\tG: -4,\tavg_r: -0.00149,\tavg_G: -6,\tt: 2681,\tdt: 5.567ms,\tSimpleTD/loss: 0.017,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00573\n",
      "INFO:TrainMonitor:ep: 277,\tT: 393,243,\tG: -5,\tavg_r: -0.00187,\tavg_G: -5.9,\tt: 2677,\tdt: 2.789ms,\tSimpleTD/loss: 0.0205,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.0069\n",
      "INFO:TrainMonitor:ep: 278,\tT: 396,236,\tG: -4,\tavg_r: -0.00134,\tavg_G: -5.71,\tt: 2992,\tdt: 2.790ms,\tSimpleTD/loss: 0.0159,\tPPOClip/EntropyRegularizer/entropy: 1.64,\tPPOClip/loss: -0.00441\n",
      "INFO:TrainMonitor:ep: 279,\tT: 399,167,\tG: -3,\tavg_r: -0.00102,\tavg_G: -5.44,\tt: 2930,\tdt: 2.798ms,\tSimpleTD/loss: 0.0162,\tPPOClip/EntropyRegularizer/entropy: 1.64,\tPPOClip/loss: -0.00581\n",
      "INFO:TrainMonitor:ep: 280,\tT: 402,281,\tG: -2,\tavg_r: -0.000642,\tavg_G: -5.09,\tt: 3113,\tdt: 5.941ms,\tSimpleTD/loss: 0.016,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00634\n",
      "INFO:TrainMonitor:ep: 281,\tT: 405,108,\tG: -3,\tavg_r: -0.00106,\tavg_G: -4.88,\tt: 2826,\tdt: 2.807ms,\tSimpleTD/loss: 0.0195,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00629\n",
      "INFO:TrainMonitor:ep: 282,\tT: 407,764,\tG: -5,\tavg_r: -0.00188,\tavg_G: -4.89,\tt: 2655,\tdt: 2.788ms,\tSimpleTD/loss: 0.0168,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00803\n",
      "INFO:TrainMonitor:ep: 283,\tT: 410,369,\tG: -5,\tavg_r: -0.00192,\tavg_G: -4.91,\tt: 2604,\tdt: 6.194ms,\tSimpleTD/loss: 0.0191,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00548\n",
      "INFO:TrainMonitor:ep: 284,\tT: 413,134,\tG: -3,\tavg_r: -0.00109,\tavg_G: -4.71,\tt: 2764,\tdt: 2.798ms,\tSimpleTD/loss: 0.0198,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00534\n",
      "INFO:TrainMonitor:ep: 285,\tT: 415,651,\tG: -5,\tavg_r: -0.00199,\tavg_G: -4.74,\tt: 2516,\tdt: 2.796ms,\tSimpleTD/loss: 0.016,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00401\n",
      "INFO:TrainMonitor:ep: 286,\tT: 418,642,\tG: -2,\tavg_r: -0.000669,\tavg_G: -4.47,\tt: 2990,\tdt: 2.807ms,\tSimpleTD/loss: 0.0132,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.0044\n",
      "INFO:TrainMonitor:ep: 287,\tT: 421,432,\tG: -4,\tavg_r: -0.00143,\tavg_G: -4.42,\tt: 2789,\tdt: 6.074ms,\tSimpleTD/loss: 0.0129,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00487\n",
      "INFO:TrainMonitor:ep: 288,\tT: 424,220,\tG: -4,\tavg_r: -0.00144,\tavg_G: -4.38,\tt: 2787,\tdt: 2.783ms,\tSimpleTD/loss: 0.0158,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00573\n",
      "INFO:TrainMonitor:ep: 289,\tT: 427,475,\tG: 1,\tavg_r: 0.000307,\tavg_G: -3.84,\tt: 3254,\tdt: 2.815ms,\tSimpleTD/loss: 0.0105,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00527\n",
      "INFO:TrainMonitor:ep: 290,\tT: 430,352,\tG: -1,\tavg_r: -0.000348,\tavg_G: -3.56,\tt: 2876,\tdt: 6.192ms,\tSimpleTD/loss: 0.0135,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00752\n",
      "INFO:TrainMonitor:ep: 291,\tT: 433,823,\tG: -1,\tavg_r: -0.000288,\tavg_G: -3.3,\tt: 3470,\tdt: 2.817ms,\tSimpleTD/loss: 0.0153,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00706\n",
      "INFO:TrainMonitor:ep: 292,\tT: 436,676,\tG: 4,\tavg_r: 0.0014,\tavg_G: -2.57,\tt: 2852,\tdt: 2.798ms,\tSimpleTD/loss: 0.0109,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00581\n",
      "INFO:TrainMonitor:ep: 293,\tT: 439,629,\tG: -1,\tavg_r: -0.000339,\tavg_G: -2.41,\tt: 2952,\tdt: 2.809ms,\tSimpleTD/loss: 0.0152,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00763\n",
      "INFO:TrainMonitor:ep: 294,\tT: 442,296,\tG: -3,\tavg_r: -0.00113,\tavg_G: -2.47,\tt: 2666,\tdt: 6.506ms,\tSimpleTD/loss: 0.0197,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00716\n",
      "INFO:TrainMonitor:ep: 295,\tT: 444,975,\tG: -3,\tavg_r: -0.00112,\tavg_G: -2.53,\tt: 2678,\tdt: 2.787ms,\tSimpleTD/loss: 0.0181,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00648\n",
      "INFO:TrainMonitor:ep: 296,\tT: 447,871,\tG: -1,\tavg_r: -0.000345,\tavg_G: -2.37,\tt: 2895,\tdt: 2.808ms,\tSimpleTD/loss: 0.0145,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00629\n",
      "INFO:TrainMonitor:ep: 297,\tT: 450,710,\tG: 3,\tavg_r: 0.00106,\tavg_G: -1.84,\tt: 2838,\tdt: 6.289ms,\tSimpleTD/loss: 0.015,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00623\n",
      "INFO:TrainMonitor:ep: 298,\tT: 453,696,\tG: 3,\tavg_r: 0.00101,\tavg_G: -1.35,\tt: 2985,\tdt: 2.804ms,\tSimpleTD/loss: 0.0182,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00789\n",
      "INFO:TrainMonitor:ep: 299,\tT: 456,520,\tG: -2,\tavg_r: -0.000708,\tavg_G: -1.42,\tt: 2823,\tdt: 2.784ms,\tSimpleTD/loss: 0.0172,\tPPOClip/EntropyRegularizer/entropy: 1.64,\tPPOClip/loss: -0.00548\n",
      "INFO:TrainMonitor:ep: 300,\tT: 459,447,\tG: -2,\tavg_r: -0.000684,\tavg_G: -1.48,\tt: 2926,\tdt: 2.786ms,\tSimpleTD/loss: 0.0185,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00725\n",
      "INFO:TrainMonitor:ep: 301,\tT: 461,903,\tG: -4,\tavg_r: -0.00163,\tavg_G: -1.73,\tt: 2455,\tdt: 5.533ms,\tSimpleTD/loss: 0.0173,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00677\n",
      "INFO:TrainMonitor:ep: 302,\tT: 464,814,\tG: -2,\tavg_r: -0.000687,\tavg_G: -1.76,\tt: 2910,\tdt: 2.789ms,\tSimpleTD/loss: 0.0212,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00675\n",
      "INFO:TrainMonitor:ep: 303,\tT: 467,658,\tG: 1,\tavg_r: 0.000352,\tavg_G: -1.48,\tt: 2843,\tdt: 2.798ms,\tSimpleTD/loss: 0.022,\tPPOClip/EntropyRegularizer/entropy: 1.64,\tPPOClip/loss: -0.00802\n",
      "INFO:TrainMonitor:ep: 304,\tT: 470,483,\tG: -4,\tavg_r: -0.00142,\tavg_G: -1.73,\tt: 2824,\tdt: 6.162ms,\tSimpleTD/loss: 0.0227,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00886\n",
      "INFO:TrainMonitor:ep: 305,\tT: 473,117,\tG: -5,\tavg_r: -0.0019,\tavg_G: -2.06,\tt: 2633,\tdt: 2.802ms,\tSimpleTD/loss: 0.0188,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00706\n",
      "INFO:TrainMonitor:ep: 306,\tT: 475,705,\tG: -8,\tavg_r: -0.00309,\tavg_G: -2.65,\tt: 2587,\tdt: 2.801ms,\tSimpleTD/loss: 0.023,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.0083\n",
      "INFO:TrainMonitor:ep: 307,\tT: 478,458,\tG: -2,\tavg_r: -0.000727,\tavg_G: -2.59,\tt: 2752,\tdt: 2.820ms,\tSimpleTD/loss: 0.0189,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00474\n",
      "INFO:TrainMonitor:ep: 308,\tT: 481,094,\tG: -7,\tavg_r: -0.00266,\tavg_G: -3.03,\tt: 2635,\tdt: 5.857ms,\tSimpleTD/loss: 0.0186,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00798\n",
      "INFO:TrainMonitor:ep: 309,\tT: 484,095,\tG: -1,\tavg_r: -0.000333,\tavg_G: -2.83,\tt: 3000,\tdt: 2.784ms,\tSimpleTD/loss: 0.0154,\tPPOClip/EntropyRegularizer/entropy: 1.65,\tPPOClip/loss: -0.00497\n",
      "INFO:TrainMonitor:ep: 310,\tT: 486,798,\tG: -4,\tavg_r: -0.00148,\tavg_G: -2.94,\tt: 2702,\tdt: 2.807ms,\tSimpleTD/loss: 0.0126,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00659\n",
      "INFO:TrainMonitor:ep: 311,\tT: 489,339,\tG: -6,\tavg_r: -0.00236,\tavg_G: -3.25,\tt: 2540,\tdt: 2.803ms,\tSimpleTD/loss: 0.0138,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00755\n",
      "INFO:TrainMonitor:ep: 312,\tT: 492,582,\tG: 3,\tavg_r: 0.000925,\tavg_G: -2.62,\tt: 3242,\tdt: 5.519ms,\tSimpleTD/loss: 0.0151,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00564\n",
      "INFO:TrainMonitor:ep: 313,\tT: 495,243,\tG: -2,\tavg_r: -0.000752,\tavg_G: -2.56,\tt: 2660,\tdt: 2.816ms,\tSimpleTD/loss: 0.0144,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.0058\n",
      "INFO:TrainMonitor:ep: 314,\tT: 498,332,\tG: 2,\tavg_r: 0.000648,\tavg_G: -2.11,\tt: 3088,\tdt: 2.784ms,\tSimpleTD/loss: 0.0119,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00431\n",
      "INFO:TrainMonitor:ep: 315,\tT: 501,470,\tG: -1,\tavg_r: -0.000319,\tavg_G: -1.99,\tt: 3137,\tdt: 5.605ms,\tSimpleTD/loss: 0.0133,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00521\n",
      "INFO:TrainMonitor:ep: 316,\tT: 504,348,\tG: -3,\tavg_r: -0.00104,\tavg_G: -2.1,\tt: 2877,\tdt: 2.789ms,\tSimpleTD/loss: 0.0176,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00692\n",
      "INFO:TrainMonitor:ep: 317,\tT: 507,231,\tG: 2,\tavg_r: 0.000694,\tavg_G: -1.69,\tt: 2882,\tdt: 2.812ms,\tSimpleTD/loss: 0.0142,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00569\n",
      "INFO:TrainMonitor:ep: 318,\tT: 510,293,\tG: 3,\tavg_r: 0.00098,\tavg_G: -1.22,\tt: 3061,\tdt: 5.939ms,\tSimpleTD/loss: 0.0163,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00679\n",
      "INFO:TrainMonitor:ep: 319,\tT: 513,251,\tG: -3,\tavg_r: -0.00101,\tavg_G: -1.4,\tt: 2957,\tdt: 2.780ms,\tSimpleTD/loss: 0.0198,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00722\n",
      "INFO:TrainMonitor:ep: 320,\tT: 516,302,\tG: 4,\tavg_r: 0.00131,\tavg_G: -0.856,\tt: 3050,\tdt: 2.798ms,\tSimpleTD/loss: 0.0148,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00574\n",
      "INFO:TrainMonitor:ep: 321,\tT: 519,293,\tG: -2,\tavg_r: -0.000669,\tavg_G: -0.97,\tt: 2990,\tdt: 2.815ms,\tSimpleTD/loss: 0.0114,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00584\n",
      "INFO:TrainMonitor:ep: 322,\tT: 522,149,\tG: 4,\tavg_r: 0.0014,\tavg_G: -0.473,\tt: 2855,\tdt: 6.151ms,\tSimpleTD/loss: 0.0178,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00654\n",
      "INFO:TrainMonitor:ep: 323,\tT: 525,110,\tG: -1,\tavg_r: -0.000338,\tavg_G: -0.526,\tt: 2960,\tdt: 2.781ms,\tSimpleTD/loss: 0.0131,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00642\n",
      "INFO:TrainMonitor:ep: 324,\tT: 528,561,\tG: -1,\tavg_r: -0.00029,\tavg_G: -0.573,\tt: 3450,\tdt: 2.801ms,\tSimpleTD/loss: 0.0125,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00517\n",
      "INFO:TrainMonitor:ep: 325,\tT: 531,363,\tG: -3,\tavg_r: -0.00107,\tavg_G: -0.816,\tt: 2801,\tdt: 6.432ms,\tSimpleTD/loss: 0.0127,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00529\n",
      "INFO:TrainMonitor:ep: 326,\tT: 534,810,\tG: 1,\tavg_r: 0.00029,\tavg_G: -0.634,\tt: 3446,\tdt: 2.798ms,\tSimpleTD/loss: 0.0174,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00563\n",
      "INFO:TrainMonitor:ep: 327,\tT: 537,676,\tG: -2,\tavg_r: -0.000698,\tavg_G: -0.771,\tt: 2865,\tdt: 2.790ms,\tSimpleTD/loss: 0.0146,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00632\n",
      "INFO:TrainMonitor:ep: 328,\tT: 540,726,\tG: 4,\tavg_r: 0.00131,\tavg_G: -0.294,\tt: 3049,\tdt: 5.848ms,\tSimpleTD/loss: 0.0173,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00557\n",
      "INFO:TrainMonitor:ep: 329,\tT: 543,498,\tG: -6,\tavg_r: -0.00217,\tavg_G: -0.865,\tt: 2771,\tdt: 2.822ms,\tSimpleTD/loss: 0.0179,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00881\n",
      "INFO:TrainMonitor:ep: 330,\tT: 546,569,\tG: -3,\tavg_r: -0.000977,\tavg_G: -1.08,\tt: 3070,\tdt: 2.815ms,\tSimpleTD/loss: 0.0186,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00435\n",
      "INFO:TrainMonitor:ep: 331,\tT: 549,602,\tG: -1,\tavg_r: -0.00033,\tavg_G: -1.07,\tt: 3032,\tdt: 2.815ms,\tSimpleTD/loss: 0.0126,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00513\n",
      "INFO:TrainMonitor:ep: 332,\tT: 552,407,\tG: -1,\tavg_r: -0.000357,\tavg_G: -1.06,\tt: 2804,\tdt: 6.189ms,\tSimpleTD/loss: 0.0129,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00405\n",
      "INFO:TrainMonitor:ep: 333,\tT: 555,319,\tG: 1,\tavg_r: 0.000344,\tavg_G: -0.857,\tt: 2911,\tdt: 2.795ms,\tSimpleTD/loss: 0.0126,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00518\n",
      "INFO:TrainMonitor:ep: 334,\tT: 558,188,\tG: -2,\tavg_r: -0.000697,\tavg_G: -0.971,\tt: 2868,\tdt: 2.829ms,\tSimpleTD/loss: 0.0147,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00565\n",
      "INFO:TrainMonitor:ep: 335,\tT: 561,162,\tG: 3,\tavg_r: 0.00101,\tavg_G: -0.574,\tt: 2973,\tdt: 6.028ms,\tSimpleTD/loss: 0.0119,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00567\n",
      "INFO:TrainMonitor:ep: 336,\tT: 564,308,\tG: -1,\tavg_r: -0.000318,\tavg_G: -0.617,\tt: 3145,\tdt: 2.819ms,\tSimpleTD/loss: 0.0141,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00667\n",
      "INFO:TrainMonitor:ep: 337,\tT: 567,341,\tG: -1,\tavg_r: -0.00033,\tavg_G: -0.655,\tt: 3032,\tdt: 2.829ms,\tSimpleTD/loss: 0.0147,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00511\n",
      "INFO:TrainMonitor:ep: 338,\tT: 570,324,\tG: 3,\tavg_r: 0.00101,\tavg_G: -0.29,\tt: 2982,\tdt: 5.756ms,\tSimpleTD/loss: 0.0154,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00603\n",
      "INFO:TrainMonitor:ep: 339,\tT: 573,152,\tG: -2,\tavg_r: -0.000707,\tavg_G: -0.461,\tt: 2827,\tdt: 2.808ms,\tSimpleTD/loss: 0.016,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00586\n",
      "INFO:TrainMonitor:ep: 340,\tT: 576,153,\tG: -1,\tavg_r: -0.000333,\tavg_G: -0.515,\tt: 3000,\tdt: 2.832ms,\tSimpleTD/loss: 0.0128,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00551\n",
      "INFO:TrainMonitor:ep: 341,\tT: 578,847,\tG: -2,\tavg_r: -0.000743,\tavg_G: -0.663,\tt: 2693,\tdt: 2.772ms,\tSimpleTD/loss: 0.0119,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00452\n",
      "INFO:TrainMonitor:ep: 342,\tT: 581,787,\tG: -1,\tavg_r: -0.00034,\tavg_G: -0.697,\tt: 2939,\tdt: 5.680ms,\tSimpleTD/loss: 0.0166,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00595\n",
      "INFO:TrainMonitor:ep: 343,\tT: 584,894,\tG: 1,\tavg_r: 0.000322,\tavg_G: -0.527,\tt: 3106,\tdt: 2.808ms,\tSimpleTD/loss: 0.0176,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00555\n",
      "INFO:TrainMonitor:ep: 344,\tT: 587,956,\tG: 2,\tavg_r: 0.000653,\tavg_G: -0.274,\tt: 3061,\tdt: 2.801ms,\tSimpleTD/loss: 0.0145,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00641\n",
      "INFO:TrainMonitor:ep: 345,\tT: 590,913,\tG: -1,\tavg_r: -0.000338,\tavg_G: -0.347,\tt: 2956,\tdt: 5.149ms,\tSimpleTD/loss: 0.0139,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00596\n",
      "INFO:TrainMonitor:ep: 346,\tT: 593,969,\tG: -1,\tavg_r: -0.000327,\tavg_G: -0.412,\tt: 3055,\tdt: 2.791ms,\tSimpleTD/loss: 0.0149,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00444\n",
      "INFO:TrainMonitor:ep: 347,\tT: 596,996,\tG: -3,\tavg_r: -0.000991,\tavg_G: -0.671,\tt: 3026,\tdt: 2.803ms,\tSimpleTD/loss: 0.0147,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00652\n",
      "INFO:TrainMonitor:ep: 348,\tT: 599,972,\tG: -3,\tavg_r: -0.00101,\tavg_G: -0.904,\tt: 2975,\tdt: 2.800ms,\tSimpleTD/loss: 0.0175,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00534\n",
      "INFO:TrainMonitor:ep: 349,\tT: 602,921,\tG: -1,\tavg_r: -0.000339,\tavg_G: -0.914,\tt: 2948,\tdt: 5.696ms,\tSimpleTD/loss: 0.0137,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00421\n",
      "INFO:TrainMonitor:ep: 350,\tT: 605,764,\tG: -3,\tavg_r: -0.00106,\tavg_G: -1.12,\tt: 2842,\tdt: 2.795ms,\tSimpleTD/loss: 0.0152,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00684\n",
      "INFO:TrainMonitor:ep: 351,\tT: 608,743,\tG: 3,\tavg_r: 0.00101,\tavg_G: -0.71,\tt: 2978,\tdt: 2.805ms,\tSimpleTD/loss: 0.0128,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00391\n",
      "INFO:TrainMonitor:ep: 352,\tT: 611,798,\tG: -2,\tavg_r: -0.000655,\tavg_G: -0.839,\tt: 3054,\tdt: 5.970ms,\tSimpleTD/loss: 0.00949,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00513\n",
      "INFO:TrainMonitor:ep: 353,\tT: 614,550,\tG: -2,\tavg_r: -0.000727,\tavg_G: -0.955,\tt: 2751,\tdt: 2.817ms,\tSimpleTD/loss: 0.0131,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00507\n",
      "INFO:TrainMonitor:ep: 354,\tT: 617,194,\tG: 5,\tavg_r: 0.00189,\tavg_G: -0.36,\tt: 2643,\tdt: 2.789ms,\tSimpleTD/loss: 0.0163,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00657\n",
      "INFO:TrainMonitor:ep: 355,\tT: 619,925,\tG: -5,\tavg_r: -0.00183,\tavg_G: -0.824,\tt: 2730,\tdt: 2.810ms,\tSimpleTD/loss: 0.0193,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.0062\n",
      "INFO:TrainMonitor:ep: 356,\tT: 622,522,\tG: -5,\tavg_r: -0.00193,\tavg_G: -1.24,\tt: 2596,\tdt: 6.179ms,\tSimpleTD/loss: 0.0227,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00618\n",
      "INFO:TrainMonitor:ep: 357,\tT: 625,544,\tG: -2,\tavg_r: -0.000662,\tavg_G: -1.32,\tt: 3021,\tdt: 2.800ms,\tSimpleTD/loss: 0.0192,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00523\n",
      "INFO:TrainMonitor:ep: 358,\tT: 628,346,\tG: -2,\tavg_r: -0.000714,\tavg_G: -1.39,\tt: 2801,\tdt: 2.794ms,\tSimpleTD/loss: 0.0225,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.0075\n",
      "INFO:TrainMonitor:ep: 359,\tT: 631,179,\tG: -2,\tavg_r: -0.000706,\tavg_G: -1.45,\tt: 2832,\tdt: 5.775ms,\tSimpleTD/loss: 0.0181,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00431\n",
      "INFO:TrainMonitor:ep: 360,\tT: 634,237,\tG: -3,\tavg_r: -0.000981,\tavg_G: -1.6,\tt: 3057,\tdt: 2.789ms,\tSimpleTD/loss: 0.0124,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00544\n",
      "INFO:TrainMonitor:ep: 361,\tT: 636,695,\tG: -4,\tavg_r: -0.00163,\tavg_G: -1.84,\tt: 2457,\tdt: 2.764ms,\tSimpleTD/loss: 0.0193,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00487\n",
      "INFO:TrainMonitor:ep: 362,\tT: 639,467,\tG: -3,\tavg_r: -0.00108,\tavg_G: -1.96,\tt: 2771,\tdt: 2.799ms,\tSimpleTD/loss: 0.0152,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00478\n",
      "INFO:TrainMonitor:ep: 363,\tT: 642,292,\tG: -4,\tavg_r: -0.00142,\tavg_G: -2.16,\tt: 2824,\tdt: 6.230ms,\tSimpleTD/loss: 0.02,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00811\n",
      "INFO:TrainMonitor:ep: 364,\tT: 644,748,\tG: -5,\tavg_r: -0.00204,\tavg_G: -2.45,\tt: 2455,\tdt: 2.792ms,\tSimpleTD/loss: 0.0191,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00529\n",
      "INFO:TrainMonitor:ep: 365,\tT: 647,783,\tG: -1,\tavg_r: -0.00033,\tavg_G: -2.3,\tt: 3034,\tdt: 2.768ms,\tSimpleTD/loss: 0.013,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00442\n",
      "INFO:TrainMonitor:ep: 366,\tT: 650,370,\tG: 5,\tavg_r: 0.00193,\tavg_G: -1.57,\tt: 2586,\tdt: 6.556ms,\tSimpleTD/loss: 0.0166,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00643\n",
      "INFO:TrainMonitor:ep: 367,\tT: 653,471,\tG: 1,\tavg_r: 0.000323,\tavg_G: -1.31,\tt: 3100,\tdt: 2.782ms,\tSimpleTD/loss: 0.0162,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00611\n",
      "INFO:TrainMonitor:ep: 368,\tT: 656,262,\tG: -3,\tavg_r: -0.00108,\tavg_G: -1.48,\tt: 2790,\tdt: 2.805ms,\tSimpleTD/loss: 0.0127,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00555\n",
      "INFO:TrainMonitor:ep: 369,\tT: 659,410,\tG: 1,\tavg_r: 0.000318,\tavg_G: -1.23,\tt: 3147,\tdt: 2.798ms,\tSimpleTD/loss: 0.0112,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00549\n",
      "INFO:TrainMonitor:ep: 370,\tT: 662,400,\tG: -1,\tavg_r: -0.000335,\tavg_G: -1.21,\tt: 2989,\tdt: 5.796ms,\tSimpleTD/loss: 0.0147,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.0053\n",
      "INFO:TrainMonitor:ep: 371,\tT: 665,352,\tG: 2,\tavg_r: 0.000678,\tavg_G: -0.89,\tt: 2951,\tdt: 2.791ms,\tSimpleTD/loss: 0.0163,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00538\n",
      "INFO:TrainMonitor:ep: 372,\tT: 668,324,\tG: -1,\tavg_r: -0.000337,\tavg_G: -0.901,\tt: 2971,\tdt: 2.809ms,\tSimpleTD/loss: 0.0129,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.0051\n",
      "INFO:TrainMonitor:ep: 373,\tT: 671,375,\tG: -4,\tavg_r: -0.00131,\tavg_G: -1.21,\tt: 3050,\tdt: 5.759ms,\tSimpleTD/loss: 0.0173,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00707\n",
      "INFO:TrainMonitor:ep: 374,\tT: 674,545,\tG: -1,\tavg_r: -0.000316,\tavg_G: -1.19,\tt: 3169,\tdt: 2.791ms,\tSimpleTD/loss: 0.0129,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00493\n",
      "INFO:TrainMonitor:ep: 375,\tT: 677,482,\tG: 4,\tavg_r: 0.00136,\tavg_G: -0.671,\tt: 2936,\tdt: 2.793ms,\tSimpleTD/loss: 0.0128,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00629\n",
      "INFO:TrainMonitor:ep: 376,\tT: 680,701,\tG: 2,\tavg_r: 0.000622,\tavg_G: -0.404,\tt: 3218,\tdt: 5.526ms,\tSimpleTD/loss: 0.0141,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.0051\n",
      "INFO:TrainMonitor:ep: 377,\tT: 684,212,\tG: 1,\tavg_r: 0.000285,\tavg_G: -0.263,\tt: 3510,\tdt: 2.806ms,\tSimpleTD/loss: 0.013,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00508\n",
      "INFO:TrainMonitor:ep: 378,\tT: 687,221,\tG: 4,\tavg_r: 0.00133,\tavg_G: 0.163,\tt: 3008,\tdt: 2.765ms,\tSimpleTD/loss: 0.0202,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00668\n",
      "INFO:TrainMonitor:ep: 379,\tT: 690,128,\tG: -3,\tavg_r: -0.00103,\tavg_G: -0.153,\tt: 2906,\tdt: 6.013ms,\tSimpleTD/loss: 0.0197,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00686\n",
      "INFO:TrainMonitor:ep: 380,\tT: 693,136,\tG: -1,\tavg_r: -0.000333,\tavg_G: -0.238,\tt: 3007,\tdt: 2.810ms,\tSimpleTD/loss: 0.0163,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.0063\n",
      "INFO:TrainMonitor:ep: 381,\tT: 695,989,\tG: -2,\tavg_r: -0.000701,\tavg_G: -0.414,\tt: 2852,\tdt: 2.800ms,\tSimpleTD/loss: 0.0148,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00491\n",
      "INFO:TrainMonitor:ep: 382,\tT: 699,225,\tG: -1,\tavg_r: -0.000309,\tavg_G: -0.473,\tt: 3235,\tdt: 2.794ms,\tSimpleTD/loss: 0.0122,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00497\n",
      "INFO:TrainMonitor:ep: 383,\tT: 702,482,\tG: 1,\tavg_r: 0.000307,\tavg_G: -0.325,\tt: 3256,\tdt: 5.581ms,\tSimpleTD/loss: 0.0141,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.0071\n",
      "INFO:TrainMonitor:ep: 384,\tT: 705,615,\tG: 4,\tavg_r: 0.00128,\tavg_G: 0.107,\tt: 3132,\tdt: 2.797ms,\tSimpleTD/loss: 0.0141,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00593\n",
      "INFO:TrainMonitor:ep: 385,\tT: 708,865,\tG: -2,\tavg_r: -0.000616,\tavg_G: -0.104,\tt: 3249,\tdt: 2.801ms,\tSimpleTD/loss: 0.0153,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.0055\n",
      "INFO:TrainMonitor:ep: 386,\tT: 712,197,\tG: -1,\tavg_r: -0.0003,\tavg_G: -0.193,\tt: 3331,\tdt: 5.524ms,\tSimpleTD/loss: 0.0162,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.0059\n",
      "INFO:TrainMonitor:ep: 387,\tT: 715,453,\tG: -1,\tavg_r: -0.000307,\tavg_G: -0.274,\tt: 3255,\tdt: 2.792ms,\tSimpleTD/loss: 0.0134,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00494\n",
      "INFO:TrainMonitor:ep: 388,\tT: 718,299,\tG: -4,\tavg_r: -0.00141,\tavg_G: -0.647,\tt: 2845,\tdt: 2.807ms,\tSimpleTD/loss: 0.0161,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00693\n",
      "INFO:TrainMonitor:ep: 389,\tT: 721,285,\tG: 3,\tavg_r: 0.00101,\tavg_G: -0.282,\tt: 2985,\tdt: 6.103ms,\tSimpleTD/loss: 0.0136,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00564\n",
      "INFO:TrainMonitor:ep: 390,\tT: 724,333,\tG: 3,\tavg_r: 0.000985,\tavg_G: 0.0463,\tt: 3047,\tdt: 2.807ms,\tSimpleTD/loss: 0.0118,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00523\n",
      "INFO:TrainMonitor:ep: 391,\tT: 727,442,\tG: -2,\tavg_r: -0.000644,\tavg_G: -0.158,\tt: 3108,\tdt: 2.780ms,\tSimpleTD/loss: 0.0161,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.0073\n",
      "INFO:TrainMonitor:ep: 392,\tT: 730,482,\tG: -2,\tavg_r: -0.000658,\tavg_G: -0.342,\tt: 3039,\tdt: 5.986ms,\tSimpleTD/loss: 0.0147,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00433\n",
      "INFO:TrainMonitor:ep: 393,\tT: 733,723,\tG: -2,\tavg_r: -0.000617,\tavg_G: -0.508,\tt: 3240,\tdt: 2.802ms,\tSimpleTD/loss: 0.0117,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00488\n",
      "INFO:TrainMonitor:ep: 394,\tT: 736,753,\tG: 1,\tavg_r: 0.00033,\tavg_G: -0.357,\tt: 3029,\tdt: 2.801ms,\tSimpleTD/loss: 0.0187,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00631\n",
      "INFO:TrainMonitor:ep: 395,\tT: 739,799,\tG: -2,\tavg_r: -0.000657,\tavg_G: -0.522,\tt: 3045,\tdt: 2.796ms,\tSimpleTD/loss: 0.0128,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00576\n",
      "INFO:TrainMonitor:ep: 396,\tT: 742,831,\tG: 1,\tavg_r: 0.00033,\tavg_G: -0.37,\tt: 3031,\tdt: 5.534ms,\tSimpleTD/loss: 0.0128,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00477\n",
      "INFO:TrainMonitor:ep: 397,\tT: 745,778,\tG: 1,\tavg_r: 0.000339,\tavg_G: -0.233,\tt: 2946,\tdt: 2.806ms,\tSimpleTD/loss: 0.0189,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00629\n",
      "INFO:TrainMonitor:ep: 398,\tT: 748,813,\tG: -1,\tavg_r: -0.00033,\tavg_G: -0.309,\tt: 3034,\tdt: 2.813ms,\tSimpleTD/loss: 0.0154,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00527\n",
      "INFO:TrainMonitor:ep: 399,\tT: 752,096,\tG: -1,\tavg_r: -0.000305,\tavg_G: -0.378,\tt: 3282,\tdt: 5.457ms,\tSimpleTD/loss: 0.0225,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00912\n",
      "INFO:TrainMonitor:ep: 400,\tT: 755,215,\tG: -3,\tavg_r: -0.000962,\tavg_G: -0.641,\tt: 3118,\tdt: 2.804ms,\tSimpleTD/loss: 0.0137,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00423\n",
      "INFO:TrainMonitor:ep: 401,\tT: 758,156,\tG: -3,\tavg_r: -0.00102,\tavg_G: -0.876,\tt: 2940,\tdt: 2.792ms,\tSimpleTD/loss: 0.0168,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00499\n",
      "INFO:TrainMonitor:ep: 402,\tT: 761,149,\tG: -1,\tavg_r: -0.000334,\tavg_G: -0.889,\tt: 2992,\tdt: 5.702ms,\tSimpleTD/loss: 0.0173,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00548\n",
      "INFO:TrainMonitor:ep: 403,\tT: 764,194,\tG: 1,\tavg_r: 0.000329,\tavg_G: -0.7,\tt: 3044,\tdt: 2.815ms,\tSimpleTD/loss: 0.0201,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00574\n",
      "INFO:TrainMonitor:ep: 404,\tT: 766,953,\tG: 5,\tavg_r: 0.00181,\tavg_G: -0.13,\tt: 2758,\tdt: 2.816ms,\tSimpleTD/loss: 0.0203,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00754\n",
      "INFO:TrainMonitor:ep: 405,\tT: 769,928,\tG: -1,\tavg_r: -0.000336,\tavg_G: -0.217,\tt: 2974,\tdt: 2.795ms,\tSimpleTD/loss: 0.0161,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00522\n",
      "INFO:TrainMonitor:ep: 406,\tT: 772,875,\tG: 1,\tavg_r: 0.000339,\tavg_G: -0.0953,\tt: 2946,\tdt: 5.838ms,\tSimpleTD/loss: 0.0162,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00586\n",
      "INFO:TrainMonitor:ep: 407,\tT: 775,489,\tG: -4,\tavg_r: -0.00153,\tavg_G: -0.486,\tt: 2613,\tdt: 2.809ms,\tSimpleTD/loss: 0.0122,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00614\n",
      "INFO:TrainMonitor:ep: 408,\tT: 778,102,\tG: -3,\tavg_r: -0.00115,\tavg_G: -0.737,\tt: 2612,\tdt: 2.812ms,\tSimpleTD/loss: 0.0215,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.0074\n",
      "INFO:TrainMonitor:ep: 409,\tT: 780,786,\tG: -8,\tavg_r: -0.00298,\tavg_G: -1.46,\tt: 2683,\tdt: 6.200ms,\tSimpleTD/loss: 0.0203,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00602\n",
      "INFO:TrainMonitor:ep: 410,\tT: 783,325,\tG: -5,\tavg_r: -0.00197,\tavg_G: -1.82,\tt: 2538,\tdt: 2.827ms,\tSimpleTD/loss: 0.0168,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00714\n",
      "INFO:TrainMonitor:ep: 411,\tT: 786,347,\tG: -1,\tavg_r: -0.000331,\tavg_G: -1.74,\tt: 3021,\tdt: 2.819ms,\tSimpleTD/loss: 0.0114,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00465\n",
      "INFO:TrainMonitor:ep: 412,\tT: 789,156,\tG: -4,\tavg_r: -0.00142,\tavg_G: -1.96,\tt: 2808,\tdt: 2.777ms,\tSimpleTD/loss: 0.0136,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00482\n",
      "INFO:TrainMonitor:ep: 413,\tT: 792,058,\tG: 3,\tavg_r: 0.00103,\tavg_G: -1.47,\tt: 2901,\tdt: 5.900ms,\tSimpleTD/loss: 0.0103,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00502\n",
      "INFO:TrainMonitor:ep: 414,\tT: 794,949,\tG: -3,\tavg_r: -0.00104,\tavg_G: -1.62,\tt: 2890,\tdt: 2.817ms,\tSimpleTD/loss: 0.0141,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00671\n",
      "INFO:TrainMonitor:ep: 415,\tT: 798,060,\tG: -1,\tavg_r: -0.000322,\tavg_G: -1.56,\tt: 3110,\tdt: 2.799ms,\tSimpleTD/loss: 0.0122,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00439\n",
      "INFO:TrainMonitor:ep: 416,\tT: 800,956,\tG: -2,\tavg_r: -0.000691,\tavg_G: -1.6,\tt: 2895,\tdt: 6.261ms,\tSimpleTD/loss: 0.013,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00397\n",
      "INFO:TrainMonitor:ep: 417,\tT: 804,207,\tG: -1,\tavg_r: -0.000308,\tavg_G: -1.54,\tt: 3250,\tdt: 2.789ms,\tSimpleTD/loss: 0.0149,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00849\n",
      "INFO:TrainMonitor:ep: 418,\tT: 807,414,\tG: 2,\tavg_r: 0.000624,\tavg_G: -1.19,\tt: 3206,\tdt: 2.814ms,\tSimpleTD/loss: 0.0128,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0057\n",
      "INFO:TrainMonitor:ep: 419,\tT: 810,227,\tG: 3,\tavg_r: 0.00107,\tavg_G: -0.768,\tt: 2812,\tdt: 6.299ms,\tSimpleTD/loss: 0.0132,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00662\n",
      "INFO:TrainMonitor:ep: 420,\tT: 812,836,\tG: 5,\tavg_r: 0.00192,\tavg_G: -0.192,\tt: 2608,\tdt: 2.799ms,\tSimpleTD/loss: 0.0181,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00558\n",
      "INFO:TrainMonitor:ep: 421,\tT: 815,695,\tG: 2,\tavg_r: 0.0007,\tavg_G: 0.0275,\tt: 2858,\tdt: 2.788ms,\tSimpleTD/loss: 0.0168,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00739\n",
      "INFO:TrainMonitor:ep: 422,\tT: 818,527,\tG: 4,\tavg_r: 0.00141,\tavg_G: 0.425,\tt: 2831,\tdt: 2.799ms,\tSimpleTD/loss: 0.0198,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00856\n",
      "INFO:TrainMonitor:ep: 423,\tT: 821,458,\tG: 3,\tavg_r: 0.00102,\tavg_G: 0.682,\tt: 2930,\tdt: 6.122ms,\tSimpleTD/loss: 0.0142,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00507\n",
      "INFO:TrainMonitor:ep: 424,\tT: 824,245,\tG: 6,\tavg_r: 0.00215,\tavg_G: 1.21,\tt: 2786,\tdt: 2.777ms,\tSimpleTD/loss: 0.0207,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00835\n",
      "INFO:TrainMonitor:ep: 425,\tT: 827,283,\tG: 4,\tavg_r: 0.00132,\tavg_G: 1.49,\tt: 3037,\tdt: 2.782ms,\tSimpleTD/loss: 0.0152,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00589\n",
      "INFO:TrainMonitor:ep: 426,\tT: 830,332,\tG: 2,\tavg_r: 0.000656,\tavg_G: 1.54,\tt: 3048,\tdt: 5.648ms,\tSimpleTD/loss: 0.0206,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00756\n",
      "INFO:TrainMonitor:ep: 427,\tT: 833,330,\tG: -2,\tavg_r: -0.000667,\tavg_G: 1.19,\tt: 2997,\tdt: 2.806ms,\tSimpleTD/loss: 0.0156,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00669\n",
      "INFO:TrainMonitor:ep: 428,\tT: 836,345,\tG: 2,\tavg_r: 0.000664,\tavg_G: 1.27,\tt: 3014,\tdt: 2.805ms,\tSimpleTD/loss: 0.0164,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00657\n",
      "INFO:TrainMonitor:ep: 429,\tT: 839,888,\tG: 2,\tavg_r: 0.000565,\tavg_G: 1.34,\tt: 3542,\tdt: 2.800ms,\tSimpleTD/loss: 0.0135,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00677\n",
      "INFO:TrainMonitor:ep: 430,\tT: 842,988,\tG: 4,\tavg_r: 0.00129,\tavg_G: 1.61,\tt: 3099,\tdt: 5.787ms,\tSimpleTD/loss: 0.0163,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00713\n",
      "INFO:TrainMonitor:ep: 431,\tT: 845,813,\tG: 4,\tavg_r: 0.00142,\tavg_G: 1.85,\tt: 2824,\tdt: 2.825ms,\tSimpleTD/loss: 0.0119,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00533\n",
      "INFO:TrainMonitor:ep: 432,\tT: 848,888,\tG: 2,\tavg_r: 0.000651,\tavg_G: 1.86,\tt: 3074,\tdt: 2.814ms,\tSimpleTD/loss: 0.0172,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00692\n",
      "INFO:TrainMonitor:ep: 433,\tT: 851,954,\tG: 1,\tavg_r: 0.000326,\tavg_G: 1.78,\tt: 3065,\tdt: 5.773ms,\tSimpleTD/loss: 0.0152,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00677\n",
      "INFO:TrainMonitor:ep: 434,\tT: 854,713,\tG: 4,\tavg_r: 0.00145,\tavg_G: 2,\tt: 2758,\tdt: 2.773ms,\tSimpleTD/loss: 0.0185,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00788\n",
      "INFO:TrainMonitor:ep: 435,\tT: 857,571,\tG: 2,\tavg_r: 0.0007,\tavg_G: 2,\tt: 2857,\tdt: 2.833ms,\tSimpleTD/loss: 0.0192,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00891\n",
      "INFO:TrainMonitor:ep: 436,\tT: 860,608,\tG: 3,\tavg_r: 0.000988,\tavg_G: 2.1,\tt: 3036,\tdt: 5.685ms,\tSimpleTD/loss: 0.0198,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00983\n",
      "INFO:TrainMonitor:ep: 437,\tT: 863,792,\tG: 1,\tavg_r: 0.000314,\tavg_G: 1.99,\tt: 3183,\tdt: 2.824ms,\tSimpleTD/loss: 0.016,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00575\n",
      "INFO:TrainMonitor:ep: 438,\tT: 866,829,\tG: 1,\tavg_r: 0.000329,\tavg_G: 1.89,\tt: 3036,\tdt: 2.799ms,\tSimpleTD/loss: 0.0211,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00903\n",
      "INFO:TrainMonitor:ep: 439,\tT: 869,501,\tG: 8,\tavg_r: 0.003,\tavg_G: 2.5,\tt: 2671,\tdt: 2.785ms,\tSimpleTD/loss: 0.0127,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00423\n",
      "INFO:TrainMonitor:ep: 440,\tT: 872,174,\tG: 11,\tavg_r: 0.00412,\tavg_G: 3.35,\tt: 2672,\tdt: 6.142ms,\tSimpleTD/loss: 0.0144,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00632\n",
      "INFO:TrainMonitor:ep: 441,\tT: 875,134,\tG: 3,\tavg_r: 0.00101,\tavg_G: 3.32,\tt: 2959,\tdt: 2.777ms,\tSimpleTD/loss: 0.0154,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.006\n",
      "INFO:TrainMonitor:ep: 442,\tT: 877,950,\tG: 5,\tavg_r: 0.00178,\tavg_G: 3.48,\tt: 2815,\tdt: 2.787ms,\tSimpleTD/loss: 0.0158,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00544\n",
      "INFO:TrainMonitor:ep: 443,\tT: 880,494,\tG: 7,\tavg_r: 0.00275,\tavg_G: 3.84,\tt: 2543,\tdt: 5.967ms,\tSimpleTD/loss: 0.0137,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00856\n",
      "INFO:TrainMonitor:ep: 444,\tT: 883,385,\tG: 7,\tavg_r: 0.00242,\tavg_G: 4.15,\tt: 2890,\tdt: 2.812ms,\tSimpleTD/loss: 0.0144,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00641\n",
      "INFO:TrainMonitor:ep: 445,\tT: 886,293,\tG: 5,\tavg_r: 0.00172,\tavg_G: 4.24,\tt: 2907,\tdt: 2.814ms,\tSimpleTD/loss: 0.0138,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00578\n",
      "INFO:TrainMonitor:ep: 446,\tT: 889,013,\tG: 7,\tavg_r: 0.00257,\tavg_G: 4.51,\tt: 2719,\tdt: 2.782ms,\tSimpleTD/loss: 0.0163,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00818\n",
      "INFO:TrainMonitor:ep: 447,\tT: 891,941,\tG: 3,\tavg_r: 0.00102,\tavg_G: 4.36,\tt: 2927,\tdt: 6.014ms,\tSimpleTD/loss: 0.0174,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00759\n",
      "INFO:TrainMonitor:ep: 448,\tT: 894,956,\tG: 7,\tavg_r: 0.00232,\tavg_G: 4.63,\tt: 3014,\tdt: 2.819ms,\tSimpleTD/loss: 0.0161,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00627\n",
      "INFO:TrainMonitor:ep: 449,\tT: 897,958,\tG: 6,\tavg_r: 0.002,\tavg_G: 4.76,\tt: 3001,\tdt: 2.785ms,\tSimpleTD/loss: 0.0172,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00672\n",
      "INFO:TrainMonitor:ep: 450,\tT: 900,715,\tG: 6,\tavg_r: 0.00218,\tavg_G: 4.89,\tt: 2756,\tdt: 6.230ms,\tSimpleTD/loss: 0.0147,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00752\n",
      "INFO:TrainMonitor:ep: 451,\tT: 903,390,\tG: 6,\tavg_r: 0.00224,\tavg_G: 5,\tt: 2674,\tdt: 2.815ms,\tSimpleTD/loss: 0.0164,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.0069\n",
      "INFO:TrainMonitor:ep: 452,\tT: 906,412,\tG: 5,\tavg_r: 0.00166,\tavg_G: 5,\tt: 3021,\tdt: 2.786ms,\tSimpleTD/loss: 0.0142,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00708\n",
      "INFO:TrainMonitor:ep: 453,\tT: 909,207,\tG: 4,\tavg_r: 0.00143,\tavg_G: 4.9,\tt: 2794,\tdt: 2.799ms,\tSimpleTD/loss: 0.0196,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00785\n",
      "INFO:TrainMonitor:ep: 454,\tT: 911,744,\tG: 12,\tavg_r: 0.00473,\tavg_G: 5.61,\tt: 2536,\tdt: 5.554ms,\tSimpleTD/loss: 0.0108,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00482\n",
      "INFO:TrainMonitor:ep: 455,\tT: 915,145,\tG: 1,\tavg_r: 0.000294,\tavg_G: 5.15,\tt: 3400,\tdt: 2.801ms,\tSimpleTD/loss: 0.0136,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0052\n",
      "INFO:TrainMonitor:ep: 456,\tT: 918,389,\tG: 1,\tavg_r: 0.000308,\tavg_G: 4.73,\tt: 3243,\tdt: 2.831ms,\tSimpleTD/loss: 0.0149,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00512\n",
      "INFO:TrainMonitor:ep: 457,\tT: 921,031,\tG: 9,\tavg_r: 0.00341,\tavg_G: 5.16,\tt: 2641,\tdt: 6.292ms,\tSimpleTD/loss: 0.0181,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00746\n",
      "INFO:TrainMonitor:ep: 458,\tT: 923,607,\tG: 10,\tavg_r: 0.00388,\tavg_G: 5.64,\tt: 2575,\tdt: 2.789ms,\tSimpleTD/loss: 0.0114,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00447\n",
      "INFO:TrainMonitor:ep: 459,\tT: 926,214,\tG: 9,\tavg_r: 0.00345,\tavg_G: 5.98,\tt: 2606,\tdt: 2.853ms,\tSimpleTD/loss: 0.0129,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00628\n",
      "INFO:TrainMonitor:ep: 460,\tT: 928,954,\tG: 5,\tavg_r: 0.00183,\tavg_G: 5.88,\tt: 2739,\tdt: 2.759ms,\tSimpleTD/loss: 0.0122,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00635\n",
      "INFO:TrainMonitor:ep: 461,\tT: 931,409,\tG: 12,\tavg_r: 0.00489,\tavg_G: 6.49,\tt: 2454,\tdt: 5.690ms,\tSimpleTD/loss: 0.0117,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00588\n",
      "INFO:TrainMonitor:ep: 462,\tT: 933,849,\tG: 12,\tavg_r: 0.00492,\tavg_G: 7.04,\tt: 2439,\tdt: 2.805ms,\tSimpleTD/loss: 0.0159,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00614\n",
      "INFO:TrainMonitor:ep: 463,\tT: 936,565,\tG: 7,\tavg_r: 0.00258,\tavg_G: 7.04,\tt: 2715,\tdt: 2.808ms,\tSimpleTD/loss: 0.0133,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00465\n",
      "INFO:TrainMonitor:ep: 464,\tT: 939,092,\tG: 9,\tavg_r: 0.00356,\tavg_G: 7.24,\tt: 2526,\tdt: 2.810ms,\tSimpleTD/loss: 0.013,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00559\n",
      "INFO:TrainMonitor:ep: 465,\tT: 941,557,\tG: 12,\tavg_r: 0.00487,\tavg_G: 7.71,\tt: 2464,\tdt: 6.121ms,\tSimpleTD/loss: 0.0147,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00647\n",
      "INFO:TrainMonitor:ep: 466,\tT: 944,151,\tG: 9,\tavg_r: 0.00347,\tavg_G: 7.84,\tt: 2593,\tdt: 2.797ms,\tSimpleTD/loss: 0.00909,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00399\n",
      "INFO:TrainMonitor:ep: 467,\tT: 946,794,\tG: 10,\tavg_r: 0.00379,\tavg_G: 8.06,\tt: 2642,\tdt: 2.791ms,\tSimpleTD/loss: 0.0112,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00546\n",
      "INFO:TrainMonitor:ep: 468,\tT: 949,307,\tG: 9,\tavg_r: 0.00358,\tavg_G: 8.15,\tt: 2512,\tdt: 2.812ms,\tSimpleTD/loss: 0.0114,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00413\n",
      "INFO:TrainMonitor:ep: 469,\tT: 951,495,\tG: 15,\tavg_r: 0.00686,\tavg_G: 8.84,\tt: 2187,\tdt: 5.948ms,\tSimpleTD/loss: 0.00929,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00661\n",
      "INFO:TrainMonitor:ep: 470,\tT: 953,554,\tG: 17,\tavg_r: 0.00826,\tavg_G: 9.65,\tt: 2058,\tdt: 2.797ms,\tSimpleTD/loss: 0.00558,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00328\n",
      "INFO:TrainMonitor:ep: 471,\tT: 955,621,\tG: 19,\tavg_r: 0.0092,\tavg_G: 10.6,\tt: 2066,\tdt: 2.783ms,\tSimpleTD/loss: 0.00637,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00298\n",
      "INFO:TrainMonitor:ep: 472,\tT: 958,090,\tG: 14,\tavg_r: 0.00567,\tavg_G: 10.9,\tt: 2468,\tdt: 2.825ms,\tSimpleTD/loss: 0.00843,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.0072\n",
      "INFO:TrainMonitor:ep: 473,\tT: 960,504,\tG: 13,\tavg_r: 0.00539,\tavg_G: 11.1,\tt: 2413,\tdt: 6.461ms,\tSimpleTD/loss: 0.0108,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00528\n",
      "INFO:TrainMonitor:ep: 474,\tT: 963,009,\tG: 14,\tavg_r: 0.00559,\tavg_G: 11.4,\tt: 2504,\tdt: 2.758ms,\tSimpleTD/loss: 0.0116,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00621\n",
      "INFO:TrainMonitor:ep: 475,\tT: 965,232,\tG: 14,\tavg_r: 0.0063,\tavg_G: 11.7,\tt: 2222,\tdt: 2.802ms,\tSimpleTD/loss: 0.00663,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00398\n",
      "INFO:TrainMonitor:ep: 476,\tT: 967,839,\tG: 10,\tavg_r: 0.00384,\tavg_G: 11.5,\tt: 2606,\tdt: 2.795ms,\tSimpleTD/loss: 0.0124,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00588\n",
      "INFO:TrainMonitor:ep: 477,\tT: 970,192,\tG: 13,\tavg_r: 0.00553,\tavg_G: 11.7,\tt: 2352,\tdt: 6.265ms,\tSimpleTD/loss: 0.0104,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.0045\n",
      "INFO:TrainMonitor:ep: 478,\tT: 972,705,\tG: 11,\tavg_r: 0.00438,\tavg_G: 11.6,\tt: 2512,\tdt: 2.801ms,\tSimpleTD/loss: 0.0121,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00653\n",
      "INFO:TrainMonitor:ep: 479,\tT: 975,482,\tG: 7,\tavg_r: 0.00252,\tavg_G: 11.1,\tt: 2776,\tdt: 2.800ms,\tSimpleTD/loss: 0.0123,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00629\n",
      "INFO:TrainMonitor:ep: 480,\tT: 977,996,\tG: 13,\tavg_r: 0.00517,\tavg_G: 11.3,\tt: 2513,\tdt: 2.816ms,\tSimpleTD/loss: 0.0105,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00507\n",
      "INFO:TrainMonitor:ep: 481,\tT: 980,270,\tG: 15,\tavg_r: 0.0066,\tavg_G: 11.7,\tt: 2273,\tdt: 6.443ms,\tSimpleTD/loss: 0.00703,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00481\n",
      "INFO:TrainMonitor:ep: 482,\tT: 982,618,\tG: 13,\tavg_r: 0.00554,\tavg_G: 11.8,\tt: 2347,\tdt: 2.800ms,\tSimpleTD/loss: 0.00879,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.006\n",
      "INFO:TrainMonitor:ep: 483,\tT: 985,163,\tG: 12,\tavg_r: 0.00472,\tavg_G: 11.8,\tt: 2544,\tdt: 2.798ms,\tSimpleTD/loss: 0.0121,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00695\n",
      "INFO:TrainMonitor:ep: 484,\tT: 987,300,\tG: 17,\tavg_r: 0.00796,\tavg_G: 12.4,\tt: 2136,\tdt: 2.779ms,\tSimpleTD/loss: 0.00572,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00344\n",
      "INFO:TrainMonitor:ep: 485,\tT: 989,967,\tG: 7,\tavg_r: 0.00263,\tavg_G: 11.8,\tt: 2666,\tdt: 2.814ms,\tSimpleTD/loss: 0.0115,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00607\n",
      "INFO:TrainMonitor:ep: 486,\tT: 992,238,\tG: 16,\tavg_r: 0.00705,\tavg_G: 12.2,\tt: 2270,\tdt: 6.119ms,\tSimpleTD/loss: 0.00797,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00564\n",
      "INFO:TrainMonitor:ep: 487,\tT: 994,402,\tG: 17,\tavg_r: 0.00786,\tavg_G: 12.7,\tt: 2163,\tdt: 2.816ms,\tSimpleTD/loss: 0.00537,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00532\n",
      "INFO:TrainMonitor:ep: 488,\tT: 996,492,\tG: 16,\tavg_r: 0.00766,\tavg_G: 13,\tt: 2089,\tdt: 2.781ms,\tSimpleTD/loss: 0.0053,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00562\n",
      "INFO:TrainMonitor:ep: 489,\tT: 998,764,\tG: 11,\tavg_r: 0.00484,\tavg_G: 12.8,\tt: 2271,\tdt: 2.792ms,\tSimpleTD/loss: 0.0128,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00693\n",
      "INFO:TrainMonitor:ep: 490,\tT: 1,001,206,\tG: 13,\tavg_r: 0.00533,\tavg_G: 12.9,\tt: 2441,\tdt: 6.137ms,\tSimpleTD/loss: 0.0101,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00625\n",
      "INFO:TrainMonitor:ep: 491,\tT: 1,003,963,\tG: 4,\tavg_r: 0.00145,\tavg_G: 12,\tt: 2756,\tdt: 2.832ms,\tSimpleTD/loss: 0.0127,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00496\n",
      "INFO:TrainMonitor:ep: 492,\tT: 1,006,341,\tG: 11,\tavg_r: 0.00463,\tavg_G: 11.9,\tt: 2377,\tdt: 2.826ms,\tSimpleTD/loss: 0.0114,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00416\n",
      "INFO:TrainMonitor:ep: 493,\tT: 1,008,628,\tG: 14,\tavg_r: 0.00612,\tavg_G: 12.1,\tt: 2286,\tdt: 2.777ms,\tSimpleTD/loss: 0.013,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00582\n",
      "INFO:TrainMonitor:ep: 494,\tT: 1,011,223,\tG: 10,\tavg_r: 0.00386,\tavg_G: 11.9,\tt: 2594,\tdt: 5.681ms,\tSimpleTD/loss: 0.00937,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00453\n",
      "INFO:TrainMonitor:ep: 495,\tT: 1,013,598,\tG: 12,\tavg_r: 0.00505,\tavg_G: 11.9,\tt: 2374,\tdt: 2.793ms,\tSimpleTD/loss: 0.00879,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00537\n",
      "INFO:TrainMonitor:ep: 496,\tT: 1,015,944,\tG: 10,\tavg_r: 0.00426,\tavg_G: 11.7,\tt: 2345,\tdt: 2.795ms,\tSimpleTD/loss: 0.0145,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00407\n",
      "INFO:TrainMonitor:ep: 497,\tT: 1,018,075,\tG: 15,\tavg_r: 0.00704,\tavg_G: 12,\tt: 2130,\tdt: 2.789ms,\tSimpleTD/loss: 0.0117,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00603\n",
      "INFO:TrainMonitor:ep: 498,\tT: 1,020,791,\tG: 9,\tavg_r: 0.00331,\tavg_G: 11.7,\tt: 2715,\tdt: 5.251ms,\tSimpleTD/loss: 0.00691,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00299\n",
      "INFO:TrainMonitor:ep: 499,\tT: 1,022,929,\tG: 17,\tavg_r: 0.00796,\tavg_G: 12.3,\tt: 2137,\tdt: 2.781ms,\tSimpleTD/loss: 0.00522,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00446\n",
      "INFO:TrainMonitor:ep: 500,\tT: 1,024,980,\tG: 16,\tavg_r: 0.0078,\tavg_G: 12.6,\tt: 2050,\tdt: 2.793ms,\tSimpleTD/loss: 0.00989,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00517\n",
      "INFO:TrainMonitor:ep: 501,\tT: 1,027,420,\tG: 12,\tavg_r: 0.00492,\tavg_G: 12.6,\tt: 2439,\tdt: 2.814ms,\tSimpleTD/loss: 0.00761,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00301\n",
      "INFO:TrainMonitor:ep: 502,\tT: 1,029,987,\tG: 10,\tavg_r: 0.0039,\tavg_G: 12.3,\tt: 2566,\tdt: 2.788ms,\tSimpleTD/loss: 0.00834,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00441\n",
      "INFO:TrainMonitor:ep: 503,\tT: 1,031,990,\tG: 17,\tavg_r: 0.00849,\tavg_G: 12.8,\tt: 2002,\tdt: 6.998ms,\tSimpleTD/loss: 0.00646,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00435\n",
      "INFO:TrainMonitor:ep: 504,\tT: 1,034,760,\tG: 8,\tavg_r: 0.00289,\tavg_G: 12.3,\tt: 2769,\tdt: 2.809ms,\tSimpleTD/loss: 0.0143,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00578\n",
      "INFO:TrainMonitor:ep: 505,\tT: 1,037,278,\tG: 11,\tavg_r: 0.00437,\tavg_G: 12.2,\tt: 2517,\tdt: 2.762ms,\tSimpleTD/loss: 0.00877,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00488\n",
      "INFO:TrainMonitor:ep: 506,\tT: 1,039,568,\tG: 15,\tavg_r: 0.00655,\tavg_G: 12.5,\tt: 2289,\tdt: 2.809ms,\tSimpleTD/loss: 0.00866,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00612\n",
      "INFO:TrainMonitor:ep: 507,\tT: 1,041,823,\tG: 15,\tavg_r: 0.00665,\tavg_G: 12.7,\tt: 2254,\tdt: 5.940ms,\tSimpleTD/loss: 0.00569,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00203\n",
      "INFO:TrainMonitor:ep: 508,\tT: 1,044,059,\tG: 16,\tavg_r: 0.00716,\tavg_G: 13,\tt: 2235,\tdt: 2.831ms,\tSimpleTD/loss: 0.00719,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0044\n",
      "INFO:TrainMonitor:ep: 509,\tT: 1,046,547,\tG: 11,\tavg_r: 0.00442,\tavg_G: 12.8,\tt: 2487,\tdt: 2.821ms,\tSimpleTD/loss: 0.00963,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00401\n",
      "INFO:TrainMonitor:ep: 510,\tT: 1,048,753,\tG: 14,\tavg_r: 0.00635,\tavg_G: 13,\tt: 2205,\tdt: 2.781ms,\tSimpleTD/loss: 0.0104,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00643\n",
      "INFO:TrainMonitor:ep: 511,\tT: 1,050,831,\tG: 17,\tavg_r: 0.00818,\tavg_G: 13.4,\tt: 2077,\tdt: 6.218ms,\tSimpleTD/loss: 0.0034,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00242\n",
      "INFO:TrainMonitor:ep: 512,\tT: 1,053,192,\tG: 12,\tavg_r: 0.00508,\tavg_G: 13.2,\tt: 2360,\tdt: 2.797ms,\tSimpleTD/loss: 0.0109,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00576\n",
      "INFO:TrainMonitor:ep: 513,\tT: 1,055,316,\tG: 18,\tavg_r: 0.00848,\tavg_G: 13.7,\tt: 2123,\tdt: 2.840ms,\tSimpleTD/loss: 0.00611,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00404\n",
      "INFO:TrainMonitor:ep: 514,\tT: 1,057,967,\tG: 9,\tavg_r: 0.0034,\tavg_G: 13.2,\tt: 2650,\tdt: 2.800ms,\tSimpleTD/loss: 0.0108,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00545\n",
      "INFO:TrainMonitor:ep: 515,\tT: 1,060,508,\tG: 10,\tavg_r: 0.00394,\tavg_G: 12.9,\tt: 2540,\tdt: 5.729ms,\tSimpleTD/loss: 0.0125,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00438\n",
      "INFO:TrainMonitor:ep: 516,\tT: 1,062,740,\tG: 11,\tavg_r: 0.00493,\tavg_G: 12.7,\tt: 2231,\tdt: 2.823ms,\tSimpleTD/loss: 0.0124,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00493\n",
      "INFO:TrainMonitor:ep: 517,\tT: 1,065,061,\tG: 15,\tavg_r: 0.00647,\tavg_G: 12.9,\tt: 2320,\tdt: 2.807ms,\tSimpleTD/loss: 0.0114,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00678\n",
      "INFO:TrainMonitor:ep: 518,\tT: 1,067,157,\tG: 17,\tavg_r: 0.00811,\tavg_G: 13.3,\tt: 2095,\tdt: 2.794ms,\tSimpleTD/loss: 0.00547,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00427\n",
      "INFO:TrainMonitor:ep: 519,\tT: 1,069,457,\tG: 12,\tavg_r: 0.00522,\tavg_G: 13.2,\tt: 2299,\tdt: 2.813ms,\tSimpleTD/loss: 0.0115,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00571\n",
      "INFO:TrainMonitor:ep: 520,\tT: 1,071,666,\tG: 14,\tavg_r: 0.00634,\tavg_G: 13.3,\tt: 2208,\tdt: 6.991ms,\tSimpleTD/loss: 0.00726,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00439\n",
      "INFO:TrainMonitor:ep: 521,\tT: 1,073,948,\tG: 13,\tavg_r: 0.0057,\tavg_G: 13.3,\tt: 2281,\tdt: 2.808ms,\tSimpleTD/loss: 0.00993,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00573\n",
      "INFO:TrainMonitor:ep: 522,\tT: 1,076,942,\tG: 6,\tavg_r: 0.002,\tavg_G: 12.5,\tt: 2993,\tdt: 2.787ms,\tSimpleTD/loss: 0.00955,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00503\n",
      "INFO:TrainMonitor:ep: 523,\tT: 1,079,250,\tG: 15,\tavg_r: 0.0065,\tavg_G: 12.8,\tt: 2307,\tdt: 2.809ms,\tSimpleTD/loss: 0.0073,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00421\n",
      "INFO:TrainMonitor:ep: 524,\tT: 1,081,328,\tG: 19,\tavg_r: 0.00915,\tavg_G: 13.4,\tt: 2077,\tdt: 5.680ms,\tSimpleTD/loss: 0.0044,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00391\n",
      "INFO:TrainMonitor:ep: 525,\tT: 1,083,318,\tG: 17,\tavg_r: 0.00855,\tavg_G: 13.8,\tt: 1989,\tdt: 2.816ms,\tSimpleTD/loss: 0.00536,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00493\n",
      "INFO:TrainMonitor:ep: 526,\tT: 1,085,622,\tG: 15,\tavg_r: 0.00651,\tavg_G: 13.9,\tt: 2303,\tdt: 2.810ms,\tSimpleTD/loss: 0.00754,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00433\n",
      "INFO:TrainMonitor:ep: 527,\tT: 1,087,660,\tG: 19,\tavg_r: 0.00933,\tavg_G: 14.4,\tt: 2037,\tdt: 2.812ms,\tSimpleTD/loss: 0.00285,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00311\n",
      "INFO:TrainMonitor:ep: 528,\tT: 1,089,629,\tG: 19,\tavg_r: 0.00965,\tavg_G: 14.9,\tt: 1968,\tdt: 2.825ms,\tSimpleTD/loss: 0.00427,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00411\n",
      "INFO:TrainMonitor:ep: 529,\tT: 1,091,517,\tG: 19,\tavg_r: 0.0101,\tavg_G: 15.3,\tt: 1887,\tdt: 6.181ms,\tSimpleTD/loss: 0.00367,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00298\n",
      "INFO:TrainMonitor:ep: 530,\tT: 1,093,427,\tG: 19,\tavg_r: 0.00995,\tavg_G: 15.6,\tt: 1909,\tdt: 2.830ms,\tSimpleTD/loss: 0.00369,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00312\n",
      "INFO:TrainMonitor:ep: 531,\tT: 1,095,461,\tG: 17,\tavg_r: 0.00836,\tavg_G: 15.8,\tt: 2033,\tdt: 2.758ms,\tSimpleTD/loss: 0.00496,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00475\n",
      "INFO:TrainMonitor:ep: 532,\tT: 1,097,636,\tG: 16,\tavg_r: 0.00736,\tavg_G: 15.8,\tt: 2174,\tdt: 2.825ms,\tSimpleTD/loss: 0.0063,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00385\n",
      "INFO:TrainMonitor:ep: 533,\tT: 1,099,666,\tG: 19,\tavg_r: 0.00936,\tavg_G: 16.1,\tt: 2029,\tdt: 2.799ms,\tSimpleTD/loss: 0.00423,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00358\n",
      "INFO:TrainMonitor:ep: 534,\tT: 1,102,759,\tG: 8,\tavg_r: 0.00259,\tavg_G: 15.3,\tt: 3092,\tdt: 4.926ms,\tSimpleTD/loss: 0.00921,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00626\n",
      "INFO:TrainMonitor:ep: 535,\tT: 1,104,657,\tG: 20,\tavg_r: 0.0105,\tavg_G: 15.8,\tt: 1897,\tdt: 2.783ms,\tSimpleTD/loss: 0.00434,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00162\n",
      "INFO:TrainMonitor:ep: 536,\tT: 1,106,542,\tG: 19,\tavg_r: 0.0101,\tavg_G: 16.1,\tt: 1884,\tdt: 2.840ms,\tSimpleTD/loss: 0.00306,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.003\n",
      "INFO:TrainMonitor:ep: 537,\tT: 1,109,493,\tG: 4,\tavg_r: 0.00136,\tavg_G: 14.9,\tt: 2950,\tdt: 2.791ms,\tSimpleTD/loss: 0.00936,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00415\n",
      "INFO:TrainMonitor:ep: 538,\tT: 1,111,647,\tG: 17,\tavg_r: 0.0079,\tavg_G: 15.1,\tt: 2153,\tdt: 6.728ms,\tSimpleTD/loss: 0.00667,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00465\n",
      "INFO:TrainMonitor:ep: 539,\tT: 1,113,763,\tG: 15,\tavg_r: 0.00709,\tavg_G: 15.1,\tt: 2115,\tdt: 2.785ms,\tSimpleTD/loss: 0.00584,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00379\n",
      "INFO:TrainMonitor:ep: 540,\tT: 1,116,729,\tG: 8,\tavg_r: 0.0027,\tavg_G: 14.4,\tt: 2965,\tdt: 2.823ms,\tSimpleTD/loss: 0.00709,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00426\n",
      "INFO:TrainMonitor:ep: 541,\tT: 1,118,739,\tG: 17,\tavg_r: 0.00846,\tavg_G: 14.6,\tt: 2009,\tdt: 2.765ms,\tSimpleTD/loss: 0.00563,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00426\n",
      "INFO:TrainMonitor:ep: 542,\tT: 1,120,870,\tG: 17,\tavg_r: 0.00798,\tavg_G: 14.9,\tt: 2130,\tdt: 6.162ms,\tSimpleTD/loss: 0.0036,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.0045\n",
      "INFO:TrainMonitor:ep: 543,\tT: 1,123,094,\tG: 16,\tavg_r: 0.0072,\tavg_G: 15,\tt: 2223,\tdt: 2.811ms,\tSimpleTD/loss: 0.00518,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00424\n",
      "INFO:TrainMonitor:ep: 544,\tT: 1,125,311,\tG: 17,\tavg_r: 0.00767,\tavg_G: 15.2,\tt: 2216,\tdt: 2.814ms,\tSimpleTD/loss: 0.00352,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00352\n",
      "INFO:TrainMonitor:ep: 545,\tT: 1,127,322,\tG: 16,\tavg_r: 0.00796,\tavg_G: 15.3,\tt: 2010,\tdt: 2.805ms,\tSimpleTD/loss: 0.00582,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.0037\n",
      "INFO:TrainMonitor:ep: 546,\tT: 1,129,744,\tG: 11,\tavg_r: 0.00454,\tavg_G: 14.8,\tt: 2421,\tdt: 2.778ms,\tSimpleTD/loss: 0.00778,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00382\n",
      "INFO:TrainMonitor:ep: 547,\tT: 1,132,175,\tG: 11,\tavg_r: 0.00453,\tavg_G: 14.5,\tt: 2430,\tdt: 5.967ms,\tSimpleTD/loss: 0.00594,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00357\n",
      "INFO:TrainMonitor:ep: 548,\tT: 1,134,725,\tG: 13,\tavg_r: 0.0051,\tavg_G: 14.3,\tt: 2549,\tdt: 2.818ms,\tSimpleTD/loss: 0.00867,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00403\n",
      "INFO:TrainMonitor:ep: 549,\tT: 1,136,866,\tG: 16,\tavg_r: 0.00748,\tavg_G: 14.5,\tt: 2140,\tdt: 2.774ms,\tSimpleTD/loss: 0.00645,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00367\n",
      "INFO:TrainMonitor:ep: 550,\tT: 1,138,967,\tG: 18,\tavg_r: 0.00857,\tavg_G: 14.8,\tt: 2100,\tdt: 2.787ms,\tSimpleTD/loss: 0.00514,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.0045\n",
      "INFO:TrainMonitor:ep: 551,\tT: 1,140,967,\tG: 17,\tavg_r: 0.0085,\tavg_G: 15.1,\tt: 1999,\tdt: 6.958ms,\tSimpleTD/loss: 0.00534,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.0046\n",
      "INFO:TrainMonitor:ep: 552,\tT: 1,143,365,\tG: 10,\tavg_r: 0.00417,\tavg_G: 14.5,\tt: 2397,\tdt: 2.812ms,\tSimpleTD/loss: 0.0121,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00665\n",
      "INFO:TrainMonitor:ep: 553,\tT: 1,146,556,\tG: 4,\tavg_r: 0.00125,\tavg_G: 13.5,\tt: 3190,\tdt: 2.796ms,\tSimpleTD/loss: 0.0113,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00659\n",
      "INFO:TrainMonitor:ep: 554,\tT: 1,148,779,\tG: 17,\tavg_r: 0.00765,\tavg_G: 13.8,\tt: 2222,\tdt: 2.815ms,\tSimpleTD/loss: 0.00803,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00395\n",
      "INFO:TrainMonitor:ep: 555,\tT: 1,151,008,\tG: 15,\tavg_r: 0.00673,\tavg_G: 14,\tt: 2228,\tdt: 5.645ms,\tSimpleTD/loss: 0.00728,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00302\n",
      "INFO:TrainMonitor:ep: 556,\tT: 1,153,065,\tG: 16,\tavg_r: 0.00778,\tavg_G: 14.2,\tt: 2056,\tdt: 2.808ms,\tSimpleTD/loss: 0.00369,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00407\n",
      "INFO:TrainMonitor:ep: 557,\tT: 1,155,336,\tG: 16,\tavg_r: 0.00705,\tavg_G: 14.3,\tt: 2270,\tdt: 2.804ms,\tSimpleTD/loss: 0.00684,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00336\n",
      "INFO:TrainMonitor:ep: 558,\tT: 1,157,663,\tG: 13,\tavg_r: 0.00559,\tavg_G: 14.2,\tt: 2326,\tdt: 2.812ms,\tSimpleTD/loss: 0.0085,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00325\n",
      "INFO:TrainMonitor:ep: 559,\tT: 1,159,703,\tG: 16,\tavg_r: 0.00785,\tavg_G: 14.4,\tt: 2039,\tdt: 2.807ms,\tSimpleTD/loss: 0.00729,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00188\n",
      "INFO:TrainMonitor:ep: 560,\tT: 1,161,883,\tG: 15,\tavg_r: 0.00688,\tavg_G: 14.5,\tt: 2179,\tdt: 6.095ms,\tSimpleTD/loss: 0.00734,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00404\n",
      "INFO:TrainMonitor:ep: 561,\tT: 1,164,259,\tG: 13,\tavg_r: 0.00547,\tavg_G: 14.3,\tt: 2375,\tdt: 2.825ms,\tSimpleTD/loss: 0.0085,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00387\n",
      "INFO:TrainMonitor:ep: 562,\tT: 1,167,140,\tG: 9,\tavg_r: 0.00313,\tavg_G: 13.8,\tt: 2880,\tdt: 2.808ms,\tSimpleTD/loss: 0.00947,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.0044\n",
      "INFO:TrainMonitor:ep: 563,\tT: 1,169,384,\tG: 13,\tavg_r: 0.0058,\tavg_G: 13.7,\tt: 2243,\tdt: 2.812ms,\tSimpleTD/loss: 0.00797,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00519\n",
      "INFO:TrainMonitor:ep: 564,\tT: 1,172,057,\tG: 10,\tavg_r: 0.00374,\tavg_G: 13.3,\tt: 2672,\tdt: 5.464ms,\tSimpleTD/loss: 0.0105,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00375\n",
      "INFO:TrainMonitor:ep: 565,\tT: 1,174,635,\tG: 12,\tavg_r: 0.00466,\tavg_G: 13.2,\tt: 2577,\tdt: 2.812ms,\tSimpleTD/loss: 0.0102,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00529\n",
      "INFO:TrainMonitor:ep: 566,\tT: 1,176,528,\tG: 19,\tavg_r: 0.01,\tavg_G: 13.8,\tt: 1892,\tdt: 2.799ms,\tSimpleTD/loss: 0.00365,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00451\n",
      "INFO:TrainMonitor:ep: 567,\tT: 1,179,738,\tG: -1,\tavg_r: -0.000312,\tavg_G: 12.3,\tt: 3209,\tdt: 2.812ms,\tSimpleTD/loss: 0.00935,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.0034\n",
      "INFO:TrainMonitor:ep: 568,\tT: 1,182,626,\tG: 6,\tavg_r: 0.00208,\tavg_G: 11.7,\tt: 2887,\tdt: 5.450ms,\tSimpleTD/loss: 0.00972,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00506\n",
      "INFO:TrainMonitor:ep: 569,\tT: 1,184,748,\tG: 19,\tavg_r: 0.00896,\tavg_G: 12.4,\tt: 2121,\tdt: 2.790ms,\tSimpleTD/loss: 0.00311,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.0033\n",
      "INFO:TrainMonitor:ep: 570,\tT: 1,187,099,\tG: 16,\tavg_r: 0.00681,\tavg_G: 12.8,\tt: 2350,\tdt: 2.797ms,\tSimpleTD/loss: 0.0036,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00127\n",
      "INFO:TrainMonitor:ep: 571,\tT: 1,189,272,\tG: 16,\tavg_r: 0.00737,\tavg_G: 13.1,\tt: 2172,\tdt: 2.831ms,\tSimpleTD/loss: 0.00728,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00445\n",
      "INFO:TrainMonitor:ep: 572,\tT: 1,191,573,\tG: 11,\tavg_r: 0.00478,\tavg_G: 12.9,\tt: 2300,\tdt: 6.857ms,\tSimpleTD/loss: 0.00977,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00507\n",
      "INFO:TrainMonitor:ep: 573,\tT: 1,194,111,\tG: 8,\tavg_r: 0.00315,\tavg_G: 12.4,\tt: 2537,\tdt: 2.802ms,\tSimpleTD/loss: 0.00959,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00332\n",
      "INFO:TrainMonitor:ep: 574,\tT: 1,196,070,\tG: 16,\tavg_r: 0.00817,\tavg_G: 12.8,\tt: 1958,\tdt: 2.776ms,\tSimpleTD/loss: 0.00713,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00549\n",
      "INFO:TrainMonitor:ep: 575,\tT: 1,198,081,\tG: 18,\tavg_r: 0.00896,\tavg_G: 13.3,\tt: 2010,\tdt: 2.796ms,\tSimpleTD/loss: 0.00513,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00379\n",
      "INFO:TrainMonitor:ep: 576,\tT: 1,200,300,\tG: 14,\tavg_r: 0.00631,\tavg_G: 13.3,\tt: 2218,\tdt: 5.847ms,\tSimpleTD/loss: 0.00598,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00441\n",
      "INFO:TrainMonitor:ep: 577,\tT: 1,202,221,\tG: 17,\tavg_r: 0.00885,\tavg_G: 13.7,\tt: 1920,\tdt: 2.777ms,\tSimpleTD/loss: 0.00622,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00313\n",
      "INFO:TrainMonitor:ep: 578,\tT: 1,205,237,\tG: 4,\tavg_r: 0.00133,\tavg_G: 12.7,\tt: 3015,\tdt: 2.806ms,\tSimpleTD/loss: 0.00871,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00479\n",
      "INFO:TrainMonitor:ep: 579,\tT: 1,208,480,\tG: 3,\tavg_r: 0.000925,\tavg_G: 11.8,\tt: 3242,\tdt: 2.807ms,\tSimpleTD/loss: 0.00974,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00461\n",
      "INFO:TrainMonitor:ep: 580,\tT: 1,211,260,\tG: 9,\tavg_r: 0.00324,\tavg_G: 11.5,\tt: 2779,\tdt: 6.005ms,\tSimpleTD/loss: 0.00824,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00302\n",
      "INFO:TrainMonitor:ep: 581,\tT: 1,213,634,\tG: 13,\tavg_r: 0.00548,\tavg_G: 11.6,\tt: 2373,\tdt: 2.779ms,\tSimpleTD/loss: 0.0117,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00284\n",
      "INFO:TrainMonitor:ep: 582,\tT: 1,215,579,\tG: 16,\tavg_r: 0.00823,\tavg_G: 12.1,\tt: 1944,\tdt: 2.807ms,\tSimpleTD/loss: 0.0109,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00458\n",
      "INFO:TrainMonitor:ep: 583,\tT: 1,218,164,\tG: 6,\tavg_r: 0.00232,\tavg_G: 11.5,\tt: 2584,\tdt: 2.783ms,\tSimpleTD/loss: 0.0145,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00232\n",
      "INFO:TrainMonitor:ep: 584,\tT: 1,220,563,\tG: 11,\tavg_r: 0.00459,\tavg_G: 11.4,\tt: 2398,\tdt: 5.638ms,\tSimpleTD/loss: 0.0105,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00494\n",
      "INFO:TrainMonitor:ep: 585,\tT: 1,222,579,\tG: 15,\tavg_r: 0.00744,\tavg_G: 11.8,\tt: 2015,\tdt: 2.807ms,\tSimpleTD/loss: 0.0107,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00478\n",
      "INFO:TrainMonitor:ep: 586,\tT: 1,225,142,\tG: 10,\tavg_r: 0.0039,\tavg_G: 11.6,\tt: 2562,\tdt: 2.810ms,\tSimpleTD/loss: 0.00662,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00387\n",
      "INFO:TrainMonitor:ep: 587,\tT: 1,227,127,\tG: 16,\tavg_r: 0.00806,\tavg_G: 12,\tt: 1984,\tdt: 2.822ms,\tSimpleTD/loss: 0.00735,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00377\n",
      "INFO:TrainMonitor:ep: 588,\tT: 1,229,532,\tG: 11,\tavg_r: 0.00458,\tavg_G: 11.9,\tt: 2404,\tdt: 2.785ms,\tSimpleTD/loss: 0.0124,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00521\n",
      "INFO:TrainMonitor:ep: 589,\tT: 1,231,460,\tG: 17,\tavg_r: 0.00882,\tavg_G: 12.4,\tt: 1927,\tdt: 5.925ms,\tSimpleTD/loss: 0.0052,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00293\n",
      "INFO:TrainMonitor:ep: 590,\tT: 1,233,414,\tG: 16,\tavg_r: 0.00819,\tavg_G: 12.8,\tt: 1953,\tdt: 2.784ms,\tSimpleTD/loss: 0.00794,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00549\n",
      "INFO:TrainMonitor:ep: 591,\tT: 1,235,247,\tG: 19,\tavg_r: 0.0104,\tavg_G: 13.4,\tt: 1832,\tdt: 2.787ms,\tSimpleTD/loss: 0.00457,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00343\n",
      "INFO:TrainMonitor:ep: 592,\tT: 1,237,243,\tG: 14,\tavg_r: 0.00702,\tavg_G: 13.5,\tt: 1995,\tdt: 2.821ms,\tSimpleTD/loss: 0.0116,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00373\n",
      "INFO:TrainMonitor:ep: 593,\tT: 1,239,514,\tG: 12,\tavg_r: 0.00529,\tavg_G: 13.3,\tt: 2270,\tdt: 2.833ms,\tSimpleTD/loss: 0.0121,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00659\n",
      "INFO:TrainMonitor:ep: 594,\tT: 1,241,705,\tG: 13,\tavg_r: 0.00594,\tavg_G: 13.3,\tt: 2190,\tdt: 5.854ms,\tSimpleTD/loss: 0.0097,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00555\n",
      "INFO:TrainMonitor:ep: 595,\tT: 1,244,211,\tG: 10,\tavg_r: 0.00399,\tavg_G: 13,\tt: 2505,\tdt: 2.775ms,\tSimpleTD/loss: 0.00887,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00317\n",
      "INFO:TrainMonitor:ep: 596,\tT: 1,246,124,\tG: 18,\tavg_r: 0.00941,\tavg_G: 13.5,\tt: 1912,\tdt: 2.836ms,\tSimpleTD/loss: 0.00584,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00401\n",
      "INFO:TrainMonitor:ep: 597,\tT: 1,248,201,\tG: 12,\tavg_r: 0.00578,\tavg_G: 13.3,\tt: 2076,\tdt: 2.832ms,\tSimpleTD/loss: 0.0105,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00362\n",
      "INFO:TrainMonitor:ep: 598,\tT: 1,250,399,\tG: 12,\tavg_r: 0.00546,\tavg_G: 13.2,\tt: 2197,\tdt: 6.274ms,\tSimpleTD/loss: 0.0139,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0058\n",
      "INFO:TrainMonitor:ep: 599,\tT: 1,252,349,\tG: 16,\tavg_r: 0.00821,\tavg_G: 13.5,\tt: 1949,\tdt: 2.811ms,\tSimpleTD/loss: 0.00851,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00424\n",
      "INFO:TrainMonitor:ep: 600,\tT: 1,254,419,\tG: 14,\tavg_r: 0.00677,\tavg_G: 13.5,\tt: 2069,\tdt: 2.797ms,\tSimpleTD/loss: 0.00933,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00663\n",
      "INFO:TrainMonitor:ep: 601,\tT: 1,256,418,\tG: 17,\tavg_r: 0.00851,\tavg_G: 13.9,\tt: 1998,\tdt: 2.817ms,\tSimpleTD/loss: 0.0108,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00326\n",
      "INFO:TrainMonitor:ep: 602,\tT: 1,258,445,\tG: 16,\tavg_r: 0.0079,\tavg_G: 14.1,\tt: 2026,\tdt: 2.805ms,\tSimpleTD/loss: 0.0108,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00456\n",
      "INFO:TrainMonitor:ep: 603,\tT: 1,260,327,\tG: 16,\tavg_r: 0.00851,\tavg_G: 14.3,\tt: 1881,\tdt: 6.012ms,\tSimpleTD/loss: 0.00677,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.0033\n",
      "INFO:TrainMonitor:ep: 604,\tT: 1,262,388,\tG: 17,\tavg_r: 0.00825,\tavg_G: 14.5,\tt: 2060,\tdt: 2.808ms,\tSimpleTD/loss: 0.00754,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00472\n",
      "INFO:TrainMonitor:ep: 605,\tT: 1,264,263,\tG: 18,\tavg_r: 0.00961,\tavg_G: 14.9,\tt: 1874,\tdt: 2.842ms,\tSimpleTD/loss: 0.0039,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00324\n",
      "INFO:TrainMonitor:ep: 606,\tT: 1,266,087,\tG: 19,\tavg_r: 0.0104,\tavg_G: 15.3,\tt: 1823,\tdt: 2.788ms,\tSimpleTD/loss: 0.0028,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00198\n",
      "INFO:TrainMonitor:ep: 607,\tT: 1,268,080,\tG: 16,\tavg_r: 0.00803,\tavg_G: 15.4,\tt: 1992,\tdt: 2.775ms,\tSimpleTD/loss: 0.00622,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00294\n",
      "INFO:TrainMonitor:ep: 608,\tT: 1,269,870,\tG: 19,\tavg_r: 0.0106,\tavg_G: 15.7,\tt: 1789,\tdt: 2.811ms,\tSimpleTD/loss: 0.00489,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00437\n",
      "INFO:TrainMonitor:ep: 609,\tT: 1,271,643,\tG: 19,\tavg_r: 0.0107,\tavg_G: 16.1,\tt: 1772,\tdt: 5.991ms,\tSimpleTD/loss: 0.00304,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.0021\n",
      "INFO:TrainMonitor:ep: 610,\tT: 1,273,558,\tG: 16,\tavg_r: 0.00836,\tavg_G: 16.1,\tt: 1914,\tdt: 2.823ms,\tSimpleTD/loss: 0.00571,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00451\n",
      "INFO:TrainMonitor:ep: 611,\tT: 1,275,227,\tG: 20,\tavg_r: 0.012,\tavg_G: 16.5,\tt: 1668,\tdt: 2.803ms,\tSimpleTD/loss: 0.00194,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00271\n",
      "INFO:TrainMonitor:ep: 612,\tT: 1,277,357,\tG: 14,\tavg_r: 0.00658,\tavg_G: 16.2,\tt: 2129,\tdt: 2.869ms,\tSimpleTD/loss: 0.00657,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00499\n",
      "INFO:TrainMonitor:ep: 613,\tT: 1,279,086,\tG: 19,\tavg_r: 0.011,\tavg_G: 16.5,\tt: 1728,\tdt: 2.784ms,\tSimpleTD/loss: 0.00394,\tPPOClip/EntropyRegularizer/entropy: 1.49,\tPPOClip/loss: -0.00257\n",
      "INFO:TrainMonitor:ep: 614,\tT: 1,280,836,\tG: 18,\tavg_r: 0.0103,\tavg_G: 16.6,\tt: 1749,\tdt: 5.849ms,\tSimpleTD/loss: 0.00225,\tPPOClip/EntropyRegularizer/entropy: 1.51,\tPPOClip/loss: -0.00192\n",
      "INFO:TrainMonitor:ep: 615,\tT: 1,282,575,\tG: 19,\tavg_r: 0.0109,\tavg_G: 16.9,\tt: 1738,\tdt: 2.818ms,\tSimpleTD/loss: 0.00344,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00399\n",
      "INFO:TrainMonitor:ep: 616,\tT: 1,284,529,\tG: 16,\tavg_r: 0.00819,\tavg_G: 16.8,\tt: 1953,\tdt: 2.826ms,\tSimpleTD/loss: 0.00708,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00307\n",
      "INFO:TrainMonitor:ep: 617,\tT: 1,286,336,\tG: 19,\tavg_r: 0.0105,\tavg_G: 17,\tt: 1806,\tdt: 2.807ms,\tSimpleTD/loss: 0.00372,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.003\n",
      "INFO:TrainMonitor:ep: 618,\tT: 1,288,209,\tG: 18,\tavg_r: 0.00962,\tavg_G: 17.1,\tt: 1872,\tdt: 2.800ms,\tSimpleTD/loss: 0.00354,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00323\n",
      "INFO:TrainMonitor:ep: 619,\tT: 1,290,142,\tG: 16,\tavg_r: 0.00828,\tavg_G: 17,\tt: 1932,\tdt: 5.776ms,\tSimpleTD/loss: 0.00416,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00344\n",
      "INFO:TrainMonitor:ep: 620,\tT: 1,292,003,\tG: 17,\tavg_r: 0.00914,\tavg_G: 17,\tt: 1860,\tdt: 2.790ms,\tSimpleTD/loss: 0.004,\tPPOClip/EntropyRegularizer/entropy: 1.5,\tPPOClip/loss: -0.00238\n",
      "INFO:TrainMonitor:ep: 621,\tT: 1,293,886,\tG: 18,\tavg_r: 0.00956,\tavg_G: 17.1,\tt: 1882,\tdt: 2.813ms,\tSimpleTD/loss: 0.00685,\tPPOClip/EntropyRegularizer/entropy: 1.51,\tPPOClip/loss: -0.00104\n",
      "INFO:TrainMonitor:ep: 622,\tT: 1,295,640,\tG: 18,\tavg_r: 0.0103,\tavg_G: 17.2,\tt: 1753,\tdt: 2.805ms,\tSimpleTD/loss: 0.00332,\tPPOClip/EntropyRegularizer/entropy: 1.5,\tPPOClip/loss: -0.00456\n",
      "INFO:TrainMonitor:ep: 623,\tT: 1,297,306,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.5,\tt: 1665,\tdt: 2.821ms,\tSimpleTD/loss: 0.00159,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00188\n",
      "INFO:TrainMonitor:ep: 624,\tT: 1,299,104,\tG: 18,\tavg_r: 0.01,\tavg_G: 17.5,\tt: 1797,\tdt: 2.819ms,\tSimpleTD/loss: 0.00365,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00386\n",
      "INFO:TrainMonitor:ep: 625,\tT: 1,300,771,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.8,\tt: 1666,\tdt: 5.943ms,\tSimpleTD/loss: 0.00126,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00183\n",
      "INFO:TrainMonitor:ep: 626,\tT: 1,302,514,\tG: 19,\tavg_r: 0.0109,\tavg_G: 17.9,\tt: 1742,\tdt: 2.804ms,\tSimpleTD/loss: 0.00273,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00358\n",
      "INFO:TrainMonitor:ep: 627,\tT: 1,304,275,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18,\tt: 1760,\tdt: 2.808ms,\tSimpleTD/loss: 0.00282,\tPPOClip/EntropyRegularizer/entropy: 1.52,\tPPOClip/loss: -0.00147\n",
      "INFO:TrainMonitor:ep: 628,\tT: 1,305,996,\tG: 19,\tavg_r: 0.011,\tavg_G: 18.1,\tt: 1720,\tdt: 2.815ms,\tSimpleTD/loss: 0.00227,\tPPOClip/EntropyRegularizer/entropy: 1.52,\tPPOClip/loss: -0.00206\n",
      "INFO:TrainMonitor:ep: 629,\tT: 1,307,699,\tG: 19,\tavg_r: 0.0112,\tavg_G: 18.2,\tt: 1702,\tdt: 2.814ms,\tSimpleTD/loss: 0.00193,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00141\n",
      "INFO:TrainMonitor:ep: 630,\tT: 1,309,489,\tG: 17,\tavg_r: 0.0095,\tavg_G: 18.1,\tt: 1789,\tdt: 2.810ms,\tSimpleTD/loss: 0.00365,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00209\n",
      "INFO:TrainMonitor:ep: 631,\tT: 1,311,232,\tG: 19,\tavg_r: 0.0109,\tavg_G: 18.2,\tt: 1742,\tdt: 6.014ms,\tSimpleTD/loss: 0.00128,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00286\n",
      "INFO:TrainMonitor:ep: 632,\tT: 1,313,053,\tG: 18,\tavg_r: 0.00989,\tavg_G: 18.1,\tt: 1820,\tdt: 2.804ms,\tSimpleTD/loss: 0.00427,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00177\n",
      "INFO:TrainMonitor:ep: 633,\tT: 1,314,723,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.3,\tt: 1669,\tdt: 2.847ms,\tSimpleTD/loss: 0.00091,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.0018\n",
      "INFO:TrainMonitor:ep: 634,\tT: 1,316,621,\tG: 16,\tavg_r: 0.00843,\tavg_G: 18.1,\tt: 1897,\tdt: 2.811ms,\tSimpleTD/loss: 0.00561,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00322\n",
      "INFO:TrainMonitor:ep: 635,\tT: 1,318,412,\tG: 18,\tavg_r: 0.0101,\tavg_G: 18.1,\tt: 1790,\tdt: 2.817ms,\tSimpleTD/loss: 0.00127,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00174\n",
      "INFO:TrainMonitor:ep: 636,\tT: 1,320,485,\tG: 14,\tavg_r: 0.00676,\tavg_G: 17.7,\tt: 2072,\tdt: 5.744ms,\tSimpleTD/loss: 0.00859,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00565\n",
      "INFO:TrainMonitor:ep: 637,\tT: 1,322,303,\tG: 17,\tavg_r: 0.00936,\tavg_G: 17.6,\tt: 1817,\tdt: 2.810ms,\tSimpleTD/loss: 0.00449,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00385\n",
      "INFO:TrainMonitor:ep: 638,\tT: 1,324,425,\tG: 15,\tavg_r: 0.00707,\tavg_G: 17.4,\tt: 2121,\tdt: 2.788ms,\tSimpleTD/loss: 0.00879,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00607\n",
      "INFO:TrainMonitor:ep: 639,\tT: 1,326,477,\tG: 14,\tavg_r: 0.00683,\tavg_G: 17,\tt: 2051,\tdt: 2.791ms,\tSimpleTD/loss: 0.00428,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00278\n",
      "INFO:TrainMonitor:ep: 640,\tT: 1,328,185,\tG: 19,\tavg_r: 0.0111,\tavg_G: 17.2,\tt: 1707,\tdt: 2.822ms,\tSimpleTD/loss: 0.00387,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.000918\n",
      "INFO:TrainMonitor:ep: 641,\tT: 1,329,968,\tG: 19,\tavg_r: 0.0107,\tavg_G: 17.4,\tt: 1782,\tdt: 2.799ms,\tSimpleTD/loss: 0.00312,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00286\n",
      "INFO:TrainMonitor:ep: 642,\tT: 1,331,955,\tG: 16,\tavg_r: 0.00806,\tavg_G: 17.3,\tt: 1986,\tdt: 6.017ms,\tSimpleTD/loss: 0.00329,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00385\n",
      "INFO:TrainMonitor:ep: 643,\tT: 1,334,153,\tG: 13,\tavg_r: 0.00592,\tavg_G: 16.8,\tt: 2197,\tdt: 2.790ms,\tSimpleTD/loss: 0.00589,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00413\n",
      "INFO:TrainMonitor:ep: 644,\tT: 1,335,976,\tG: 18,\tavg_r: 0.00988,\tavg_G: 16.9,\tt: 1822,\tdt: 2.849ms,\tSimpleTD/loss: 0.00289,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00315\n",
      "INFO:TrainMonitor:ep: 645,\tT: 1,338,061,\tG: 11,\tavg_r: 0.00528,\tavg_G: 16.4,\tt: 2084,\tdt: 2.787ms,\tSimpleTD/loss: 0.0122,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00532\n",
      "INFO:TrainMonitor:ep: 646,\tT: 1,339,848,\tG: 18,\tavg_r: 0.0101,\tavg_G: 16.5,\tt: 1786,\tdt: 2.784ms,\tSimpleTD/loss: 0.0058,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00333\n",
      "INFO:TrainMonitor:ep: 647,\tT: 1,341,834,\tG: 15,\tavg_r: 0.00756,\tavg_G: 16.4,\tt: 1985,\tdt: 5.453ms,\tSimpleTD/loss: 0.00476,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00156\n",
      "INFO:TrainMonitor:ep: 648,\tT: 1,343,885,\tG: 16,\tavg_r: 0.0078,\tavg_G: 16.3,\tt: 2050,\tdt: 2.800ms,\tSimpleTD/loss: 0.0081,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00471\n",
      "INFO:TrainMonitor:ep: 649,\tT: 1,346,128,\tG: 9,\tavg_r: 0.00401,\tavg_G: 15.6,\tt: 2242,\tdt: 2.795ms,\tSimpleTD/loss: 0.00964,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00439\n",
      "INFO:TrainMonitor:ep: 650,\tT: 1,348,357,\tG: 12,\tavg_r: 0.00539,\tavg_G: 15.2,\tt: 2228,\tdt: 2.810ms,\tSimpleTD/loss: 0.00798,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00625\n",
      "INFO:TrainMonitor:ep: 651,\tT: 1,350,198,\tG: 17,\tavg_r: 0.00924,\tavg_G: 15.4,\tt: 1840,\tdt: 6.031ms,\tSimpleTD/loss: 0.00605,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00178\n",
      "INFO:TrainMonitor:ep: 652,\tT: 1,352,464,\tG: 7,\tavg_r: 0.00309,\tavg_G: 14.6,\tt: 2265,\tdt: 2.800ms,\tSimpleTD/loss: 0.0102,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0047\n",
      "INFO:TrainMonitor:ep: 653,\tT: 1,354,959,\tG: 7,\tavg_r: 0.00281,\tavg_G: 13.8,\tt: 2494,\tdt: 2.805ms,\tSimpleTD/loss: 0.0112,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00605\n",
      "INFO:TrainMonitor:ep: 654,\tT: 1,356,892,\tG: 17,\tavg_r: 0.0088,\tavg_G: 14.1,\tt: 1932,\tdt: 2.766ms,\tSimpleTD/loss: 0.00724,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00306\n",
      "INFO:TrainMonitor:ep: 655,\tT: 1,358,825,\tG: 15,\tavg_r: 0.00776,\tavg_G: 14.2,\tt: 1932,\tdt: 2.819ms,\tSimpleTD/loss: 0.00761,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00407\n",
      "INFO:TrainMonitor:ep: 656,\tT: 1,360,533,\tG: 19,\tavg_r: 0.0111,\tavg_G: 14.7,\tt: 1707,\tdt: 6.235ms,\tSimpleTD/loss: 0.0031,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00313\n",
      "INFO:TrainMonitor:ep: 657,\tT: 1,362,481,\tG: 16,\tavg_r: 0.00822,\tavg_G: 14.8,\tt: 1947,\tdt: 2.830ms,\tSimpleTD/loss: 0.0059,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00163\n",
      "INFO:TrainMonitor:ep: 658,\tT: 1,364,242,\tG: 19,\tavg_r: 0.0108,\tavg_G: 15.2,\tt: 1760,\tdt: 2.812ms,\tSimpleTD/loss: 0.0024,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00444\n",
      "INFO:TrainMonitor:ep: 659,\tT: 1,366,139,\tG: 17,\tavg_r: 0.00897,\tavg_G: 15.4,\tt: 1896,\tdt: 2.791ms,\tSimpleTD/loss: 0.0057,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00321\n",
      "INFO:TrainMonitor:ep: 660,\tT: 1,367,829,\tG: 20,\tavg_r: 0.0118,\tavg_G: 15.9,\tt: 1689,\tdt: 2.810ms,\tSimpleTD/loss: 0.00202,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00172\n",
      "INFO:TrainMonitor:ep: 661,\tT: 1,369,545,\tG: 19,\tavg_r: 0.0111,\tavg_G: 16.2,\tt: 1715,\tdt: 2.843ms,\tSimpleTD/loss: 0.00237,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00248\n",
      "INFO:TrainMonitor:ep: 662,\tT: 1,371,229,\tG: 20,\tavg_r: 0.0119,\tavg_G: 16.6,\tt: 1683,\tdt: 5.995ms,\tSimpleTD/loss: 0.00183,\tPPOClip/EntropyRegularizer/entropy: 1.52,\tPPOClip/loss: -0.00247\n",
      "INFO:TrainMonitor:ep: 663,\tT: 1,372,942,\tG: 20,\tavg_r: 0.0117,\tavg_G: 16.9,\tt: 1712,\tdt: 2.818ms,\tSimpleTD/loss: 0.00265,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00231\n",
      "INFO:TrainMonitor:ep: 664,\tT: 1,374,662,\tG: 20,\tavg_r: 0.0116,\tavg_G: 17.2,\tt: 1719,\tdt: 2.816ms,\tSimpleTD/loss: 0.00376,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.0025\n",
      "INFO:TrainMonitor:ep: 665,\tT: 1,376,558,\tG: 17,\tavg_r: 0.00897,\tavg_G: 17.2,\tt: 1895,\tdt: 2.779ms,\tSimpleTD/loss: 0.00411,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00476\n",
      "INFO:TrainMonitor:ep: 666,\tT: 1,378,370,\tG: 18,\tavg_r: 0.00994,\tavg_G: 17.3,\tt: 1811,\tdt: 2.796ms,\tSimpleTD/loss: 0.00352,\tPPOClip/EntropyRegularizer/entropy: 1.52,\tPPOClip/loss: -0.00465\n",
      "INFO:TrainMonitor:ep: 667,\tT: 1,380,154,\tG: 19,\tavg_r: 0.0107,\tavg_G: 17.5,\tt: 1783,\tdt: 5.742ms,\tSimpleTD/loss: 0.00155,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00241\n",
      "INFO:TrainMonitor:ep: 668,\tT: 1,381,817,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.7,\tt: 1662,\tdt: 2.844ms,\tSimpleTD/loss: 0.00121,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00221\n",
      "INFO:TrainMonitor:ep: 669,\tT: 1,383,562,\tG: 20,\tavg_r: 0.0115,\tavg_G: 17.9,\tt: 1744,\tdt: 2.762ms,\tSimpleTD/loss: 0.00398,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00386\n",
      "INFO:TrainMonitor:ep: 670,\tT: 1,385,322,\tG: 18,\tavg_r: 0.0102,\tavg_G: 17.9,\tt: 1759,\tdt: 2.817ms,\tSimpleTD/loss: 0.000927,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.0026\n",
      "INFO:TrainMonitor:ep: 671,\tT: 1,387,205,\tG: 17,\tavg_r: 0.00903,\tavg_G: 17.8,\tt: 1882,\tdt: 2.831ms,\tSimpleTD/loss: 0.00477,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00452\n",
      "INFO:TrainMonitor:ep: 672,\tT: 1,388,870,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.1,\tt: 1664,\tdt: 2.790ms,\tSimpleTD/loss: 0.000937,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00191\n",
      "INFO:TrainMonitor:ep: 673,\tT: 1,390,726,\tG: 17,\tavg_r: 0.00916,\tavg_G: 18,\tt: 1855,\tdt: 5.984ms,\tSimpleTD/loss: 0.00272,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00289\n",
      "INFO:TrainMonitor:ep: 674,\tT: 1,392,458,\tG: 19,\tavg_r: 0.011,\tavg_G: 18.1,\tt: 1731,\tdt: 2.806ms,\tSimpleTD/loss: 0.00417,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00323\n",
      "INFO:TrainMonitor:ep: 675,\tT: 1,394,288,\tG: 17,\tavg_r: 0.00929,\tavg_G: 18,\tt: 1829,\tdt: 2.807ms,\tSimpleTD/loss: 0.00363,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00306\n",
      "INFO:TrainMonitor:ep: 676,\tT: 1,396,014,\tG: 19,\tavg_r: 0.011,\tavg_G: 18.1,\tt: 1725,\tdt: 2.815ms,\tSimpleTD/loss: 0.00116,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00272\n",
      "INFO:TrainMonitor:ep: 677,\tT: 1,398,082,\tG: 15,\tavg_r: 0.00726,\tavg_G: 17.8,\tt: 2067,\tdt: 2.801ms,\tSimpleTD/loss: 0.00521,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00427\n",
      "INFO:TrainMonitor:ep: 678,\tT: 1,399,747,\tG: 20,\tavg_r: 0.012,\tavg_G: 18,\tt: 1664,\tdt: 2.827ms,\tSimpleTD/loss: 0.000849,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00257\n",
      "INFO:TrainMonitor:ep: 679,\tT: 1,401,506,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18.1,\tt: 1758,\tdt: 5.758ms,\tSimpleTD/loss: 0.0026,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00342\n",
      "INFO:TrainMonitor:ep: 680,\tT: 1,403,293,\tG: 18,\tavg_r: 0.0101,\tavg_G: 18.1,\tt: 1786,\tdt: 2.835ms,\tSimpleTD/loss: 0.00211,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.000831\n",
      "INFO:TrainMonitor:ep: 681,\tT: 1,404,996,\tG: 20,\tavg_r: 0.0118,\tavg_G: 18.3,\tt: 1702,\tdt: 2.821ms,\tSimpleTD/loss: 0.00361,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00364\n",
      "INFO:TrainMonitor:ep: 682,\tT: 1,406,689,\tG: 20,\tavg_r: 0.0118,\tavg_G: 18.4,\tt: 1692,\tdt: 2.815ms,\tSimpleTD/loss: 0.00137,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00209\n",
      "INFO:TrainMonitor:ep: 683,\tT: 1,408,485,\tG: 19,\tavg_r: 0.0106,\tavg_G: 18.5,\tt: 1795,\tdt: 2.808ms,\tSimpleTD/loss: 0.00161,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00296\n",
      "INFO:TrainMonitor:ep: 684,\tT: 1,410,333,\tG: 17,\tavg_r: 0.0092,\tavg_G: 18.3,\tt: 1847,\tdt: 6.818ms,\tSimpleTD/loss: 0.00325,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00248\n",
      "INFO:TrainMonitor:ep: 685,\tT: 1,412,057,\tG: 19,\tavg_r: 0.011,\tavg_G: 18.4,\tt: 1723,\tdt: 2.824ms,\tSimpleTD/loss: 0.00152,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00257\n",
      "INFO:TrainMonitor:ep: 686,\tT: 1,413,956,\tG: 15,\tavg_r: 0.0079,\tavg_G: 18.1,\tt: 1898,\tdt: 2.782ms,\tSimpleTD/loss: 0.00409,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00294\n",
      "INFO:TrainMonitor:ep: 687,\tT: 1,415,831,\tG: 18,\tavg_r: 0.00961,\tavg_G: 18.1,\tt: 1874,\tdt: 2.805ms,\tSimpleTD/loss: 0.00365,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00269\n",
      "INFO:TrainMonitor:ep: 688,\tT: 1,417,681,\tG: 17,\tavg_r: 0.00919,\tavg_G: 18,\tt: 1849,\tdt: 2.864ms,\tSimpleTD/loss: 0.00522,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.0039\n",
      "INFO:TrainMonitor:ep: 689,\tT: 1,419,712,\tG: 13,\tavg_r: 0.0064,\tavg_G: 17.5,\tt: 2030,\tdt: 2.775ms,\tSimpleTD/loss: 0.0103,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00327\n",
      "INFO:TrainMonitor:ep: 690,\tT: 1,421,616,\tG: 14,\tavg_r: 0.00736,\tavg_G: 17.1,\tt: 1903,\tdt: 5.794ms,\tSimpleTD/loss: 0.00728,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00564\n",
      "INFO:TrainMonitor:ep: 691,\tT: 1,423,281,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.4,\tt: 1664,\tdt: 2.797ms,\tSimpleTD/loss: 0.00119,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.0019\n",
      "INFO:TrainMonitor:ep: 692,\tT: 1,425,171,\tG: 17,\tavg_r: 0.009,\tavg_G: 17.4,\tt: 1889,\tdt: 2.837ms,\tSimpleTD/loss: 0.00505,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.0041\n",
      "INFO:TrainMonitor:ep: 693,\tT: 1,426,871,\tG: 20,\tavg_r: 0.0118,\tavg_G: 17.6,\tt: 1699,\tdt: 2.778ms,\tSimpleTD/loss: 0.00148,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00291\n",
      "INFO:TrainMonitor:ep: 694,\tT: 1,428,586,\tG: 21,\tavg_r: 0.0123,\tavg_G: 18,\tt: 1714,\tdt: 2.818ms,\tSimpleTD/loss: 0.00274,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.000936\n",
      "INFO:TrainMonitor:ep: 695,\tT: 1,430,383,\tG: 18,\tavg_r: 0.01,\tavg_G: 18,\tt: 1796,\tdt: 5.928ms,\tSimpleTD/loss: 0.00421,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00397\n",
      "INFO:TrainMonitor:ep: 696,\tT: 1,432,100,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.1,\tt: 1716,\tdt: 2.867ms,\tSimpleTD/loss: 0.00215,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00361\n",
      "INFO:TrainMonitor:ep: 697,\tT: 1,433,786,\tG: 20,\tavg_r: 0.0119,\tavg_G: 18.3,\tt: 1685,\tdt: 2.773ms,\tSimpleTD/loss: 0.0042,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00363\n",
      "INFO:TrainMonitor:ep: 698,\tT: 1,435,616,\tG: 18,\tavg_r: 0.00984,\tavg_G: 18.2,\tt: 1829,\tdt: 2.831ms,\tSimpleTD/loss: 0.00321,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00168\n",
      "INFO:TrainMonitor:ep: 699,\tT: 1,437,535,\tG: 18,\tavg_r: 0.00938,\tavg_G: 18.2,\tt: 1918,\tdt: 2.780ms,\tSimpleTD/loss: 0.00456,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00378\n",
      "INFO:TrainMonitor:ep: 700,\tT: 1,439,200,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.4,\tt: 1664,\tdt: 2.765ms,\tSimpleTD/loss: 0.00143,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00268\n",
      "INFO:TrainMonitor:ep: 701,\tT: 1,440,908,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.5,\tt: 1707,\tdt: 5.885ms,\tSimpleTD/loss: 0.00134,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00282\n",
      "INFO:TrainMonitor:ep: 702,\tT: 1,442,802,\tG: 15,\tavg_r: 0.00792,\tavg_G: 18.1,\tt: 1893,\tdt: 2.820ms,\tSimpleTD/loss: 0.004,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00263\n",
      "INFO:TrainMonitor:ep: 703,\tT: 1,444,929,\tG: 14,\tavg_r: 0.00659,\tavg_G: 17.7,\tt: 2126,\tdt: 2.779ms,\tSimpleTD/loss: 0.00641,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00334\n",
      "INFO:TrainMonitor:ep: 704,\tT: 1,446,591,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.9,\tt: 1661,\tdt: 2.786ms,\tSimpleTD/loss: 0.000801,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00214\n",
      "INFO:TrainMonitor:ep: 705,\tT: 1,448,405,\tG: 17,\tavg_r: 0.00938,\tavg_G: 17.8,\tt: 1813,\tdt: 2.791ms,\tSimpleTD/loss: 0.00307,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00268\n",
      "INFO:TrainMonitor:ep: 706,\tT: 1,450,264,\tG: 17,\tavg_r: 0.00915,\tavg_G: 17.8,\tt: 1858,\tdt: 5.646ms,\tSimpleTD/loss: 0.00357,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00318\n",
      "INFO:TrainMonitor:ep: 707,\tT: 1,452,120,\tG: 16,\tavg_r: 0.00863,\tavg_G: 17.6,\tt: 1855,\tdt: 2.803ms,\tSimpleTD/loss: 0.00393,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00251\n",
      "INFO:TrainMonitor:ep: 708,\tT: 1,454,071,\tG: 17,\tavg_r: 0.00872,\tavg_G: 17.5,\tt: 1950,\tdt: 2.819ms,\tSimpleTD/loss: 0.00508,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00443\n",
      "INFO:TrainMonitor:ep: 709,\tT: 1,455,741,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.8,\tt: 1669,\tdt: 2.771ms,\tSimpleTD/loss: 0.00256,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00311\n",
      "INFO:TrainMonitor:ep: 710,\tT: 1,457,413,\tG: 20,\tavg_r: 0.012,\tavg_G: 18,\tt: 1671,\tdt: 2.825ms,\tSimpleTD/loss: 0.000909,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00227\n",
      "INFO:TrainMonitor:ep: 711,\tT: 1,459,319,\tG: 16,\tavg_r: 0.0084,\tavg_G: 17.8,\tt: 1905,\tdt: 2.816ms,\tSimpleTD/loss: 0.00433,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00328\n",
      "INFO:TrainMonitor:ep: 712,\tT: 1,461,064,\tG: 19,\tavg_r: 0.0109,\tavg_G: 17.9,\tt: 1744,\tdt: 6.376ms,\tSimpleTD/loss: 0.00445,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00611\n",
      "INFO:TrainMonitor:ep: 713,\tT: 1,462,732,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.1,\tt: 1667,\tdt: 2.797ms,\tSimpleTD/loss: 0.00148,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0017\n",
      "INFO:TrainMonitor:ep: 714,\tT: 1,464,365,\tG: 21,\tavg_r: 0.0129,\tavg_G: 18.4,\tt: 1632,\tdt: 2.832ms,\tSimpleTD/loss: 0.00209,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00256\n",
      "INFO:TrainMonitor:ep: 715,\tT: 1,466,353,\tG: 17,\tavg_r: 0.00856,\tavg_G: 18.3,\tt: 1987,\tdt: 2.835ms,\tSimpleTD/loss: 0.00492,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00388\n",
      "INFO:TrainMonitor:ep: 716,\tT: 1,468,081,\tG: 19,\tavg_r: 0.011,\tavg_G: 18.3,\tt: 1727,\tdt: 2.779ms,\tSimpleTD/loss: 0.00249,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00333\n",
      "INFO:TrainMonitor:ep: 717,\tT: 1,469,748,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.5,\tt: 1666,\tdt: 2.826ms,\tSimpleTD/loss: 0.000518,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.0022\n",
      "INFO:TrainMonitor:ep: 718,\tT: 1,471,416,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.7,\tt: 1667,\tdt: 5.927ms,\tSimpleTD/loss: 0.000596,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00232\n",
      "INFO:TrainMonitor:ep: 719,\tT: 1,473,087,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.8,\tt: 1670,\tdt: 2.825ms,\tSimpleTD/loss: 0.000511,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00238\n",
      "INFO:TrainMonitor:ep: 720,\tT: 1,474,916,\tG: 19,\tavg_r: 0.0104,\tavg_G: 18.8,\tt: 1828,\tdt: 2.795ms,\tSimpleTD/loss: 0.00267,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00338\n",
      "INFO:TrainMonitor:ep: 721,\tT: 1,476,807,\tG: 17,\tavg_r: 0.00899,\tavg_G: 18.6,\tt: 1890,\tdt: 2.798ms,\tSimpleTD/loss: 0.00465,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00202\n",
      "INFO:TrainMonitor:ep: 722,\tT: 1,478,473,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.8,\tt: 1665,\tdt: 2.825ms,\tSimpleTD/loss: 0.00103,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00285\n",
      "INFO:TrainMonitor:ep: 723,\tT: 1,480,268,\tG: 19,\tavg_r: 0.0106,\tavg_G: 18.8,\tt: 1794,\tdt: 6.417ms,\tSimpleTD/loss: 0.00353,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00296\n",
      "INFO:TrainMonitor:ep: 724,\tT: 1,481,935,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.9,\tt: 1666,\tdt: 2.782ms,\tSimpleTD/loss: 0.000712,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00213\n",
      "INFO:TrainMonitor:ep: 725,\tT: 1,483,784,\tG: 17,\tavg_r: 0.0092,\tavg_G: 18.7,\tt: 1848,\tdt: 2.844ms,\tSimpleTD/loss: 0.00285,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00204\n",
      "INFO:TrainMonitor:ep: 726,\tT: 1,485,916,\tG: 12,\tavg_r: 0.00563,\tavg_G: 18,\tt: 2131,\tdt: 2.796ms,\tSimpleTD/loss: 0.0064,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00487\n",
      "INFO:TrainMonitor:ep: 727,\tT: 1,487,731,\tG: 18,\tavg_r: 0.00992,\tavg_G: 18,\tt: 1814,\tdt: 2.806ms,\tSimpleTD/loss: 0.00385,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00442\n",
      "INFO:TrainMonitor:ep: 728,\tT: 1,489,737,\tG: 12,\tavg_r: 0.00599,\tavg_G: 17.4,\tt: 2005,\tdt: 2.813ms,\tSimpleTD/loss: 0.00509,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00354\n",
      "INFO:TrainMonitor:ep: 729,\tT: 1,491,492,\tG: 18,\tavg_r: 0.0103,\tavg_G: 17.5,\tt: 1754,\tdt: 6.137ms,\tSimpleTD/loss: 0.00264,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00289\n",
      "INFO:TrainMonitor:ep: 730,\tT: 1,493,336,\tG: 17,\tavg_r: 0.00922,\tavg_G: 17.4,\tt: 1843,\tdt: 2.800ms,\tSimpleTD/loss: 0.00366,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0018\n",
      "INFO:TrainMonitor:ep: 731,\tT: 1,495,353,\tG: 14,\tavg_r: 0.00694,\tavg_G: 17.1,\tt: 2016,\tdt: 2.810ms,\tSimpleTD/loss: 0.00502,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00356\n",
      "INFO:TrainMonitor:ep: 732,\tT: 1,497,826,\tG: 6,\tavg_r: 0.00243,\tavg_G: 16,\tt: 2472,\tdt: 2.784ms,\tSimpleTD/loss: 0.00911,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00287\n",
      "INFO:TrainMonitor:ep: 733,\tT: 1,499,508,\tG: 20,\tavg_r: 0.0119,\tavg_G: 16.4,\tt: 1681,\tdt: 2.826ms,\tSimpleTD/loss: 0.00284,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.0029\n",
      "INFO:TrainMonitor:ep: 734,\tT: 1,501,838,\tG: 9,\tavg_r: 0.00386,\tavg_G: 15.7,\tt: 2329,\tdt: 5.739ms,\tSimpleTD/loss: 0.0116,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00631\n",
      "INFO:TrainMonitor:ep: 735,\tT: 1,503,602,\tG: 19,\tavg_r: 0.0108,\tavg_G: 16,\tt: 1763,\tdt: 2.837ms,\tSimpleTD/loss: 0.003,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0032\n",
      "INFO:TrainMonitor:ep: 736,\tT: 1,505,423,\tG: 17,\tavg_r: 0.00934,\tavg_G: 16.1,\tt: 1820,\tdt: 2.798ms,\tSimpleTD/loss: 0.00488,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00309\n",
      "INFO:TrainMonitor:ep: 737,\tT: 1,507,601,\tG: 14,\tavg_r: 0.00643,\tavg_G: 15.9,\tt: 2177,\tdt: 2.826ms,\tSimpleTD/loss: 0.00789,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00384\n",
      "INFO:TrainMonitor:ep: 738,\tT: 1,509,491,\tG: 15,\tavg_r: 0.00794,\tavg_G: 15.8,\tt: 1889,\tdt: 2.802ms,\tSimpleTD/loss: 0.00477,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00291\n",
      "INFO:TrainMonitor:ep: 739,\tT: 1,511,250,\tG: 18,\tavg_r: 0.0102,\tavg_G: 16,\tt: 1758,\tdt: 5.906ms,\tSimpleTD/loss: 0.00434,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00462\n",
      "INFO:TrainMonitor:ep: 740,\tT: 1,513,306,\tG: 16,\tavg_r: 0.00779,\tavg_G: 16,\tt: 2055,\tdt: 2.801ms,\tSimpleTD/loss: 0.00742,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00288\n",
      "INFO:TrainMonitor:ep: 741,\tT: 1,515,055,\tG: 20,\tavg_r: 0.0114,\tavg_G: 16.4,\tt: 1748,\tdt: 2.824ms,\tSimpleTD/loss: 0.00358,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00321\n",
      "INFO:TrainMonitor:ep: 742,\tT: 1,517,247,\tG: 14,\tavg_r: 0.00639,\tavg_G: 16.2,\tt: 2191,\tdt: 2.801ms,\tSimpleTD/loss: 0.00764,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00383\n",
      "INFO:TrainMonitor:ep: 743,\tT: 1,519,073,\tG: 17,\tavg_r: 0.00932,\tavg_G: 16.3,\tt: 1825,\tdt: 2.797ms,\tSimpleTD/loss: 0.00709,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0044\n",
      "INFO:TrainMonitor:ep: 744,\tT: 1,521,056,\tG: 16,\tavg_r: 0.00807,\tavg_G: 16.2,\tt: 1982,\tdt: 5.939ms,\tSimpleTD/loss: 0.00589,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0044\n",
      "INFO:TrainMonitor:ep: 745,\tT: 1,522,850,\tG: 18,\tavg_r: 0.01,\tavg_G: 16.4,\tt: 1793,\tdt: 2.807ms,\tSimpleTD/loss: 0.00447,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00213\n",
      "INFO:TrainMonitor:ep: 746,\tT: 1,524,517,\tG: 20,\tavg_r: 0.012,\tavg_G: 16.8,\tt: 1666,\tdt: 2.833ms,\tSimpleTD/loss: 0.00144,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00337\n",
      "INFO:TrainMonitor:ep: 747,\tT: 1,526,217,\tG: 19,\tavg_r: 0.0112,\tavg_G: 17,\tt: 1699,\tdt: 2.783ms,\tSimpleTD/loss: 0.00188,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00178\n",
      "INFO:TrainMonitor:ep: 748,\tT: 1,528,114,\tG: 17,\tavg_r: 0.00897,\tavg_G: 17,\tt: 1896,\tdt: 2.839ms,\tSimpleTD/loss: 0.00383,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00325\n",
      "INFO:TrainMonitor:ep: 749,\tT: 1,529,777,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.3,\tt: 1662,\tdt: 2.774ms,\tSimpleTD/loss: 0.000864,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00292\n",
      "INFO:TrainMonitor:ep: 750,\tT: 1,531,611,\tG: 17,\tavg_r: 0.00927,\tavg_G: 17.3,\tt: 1833,\tdt: 5.668ms,\tSimpleTD/loss: 0.00308,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00303\n",
      "INFO:TrainMonitor:ep: 751,\tT: 1,533,285,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.5,\tt: 1673,\tdt: 2.816ms,\tSimpleTD/loss: 0.00104,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00269\n",
      "INFO:TrainMonitor:ep: 752,\tT: 1,535,222,\tG: 14,\tavg_r: 0.00723,\tavg_G: 17.2,\tt: 1936,\tdt: 2.784ms,\tSimpleTD/loss: 0.0063,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00232\n",
      "INFO:TrainMonitor:ep: 753,\tT: 1,537,272,\tG: 13,\tavg_r: 0.00634,\tavg_G: 16.8,\tt: 2049,\tdt: 2.807ms,\tSimpleTD/loss: 0.00913,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00433\n",
      "INFO:TrainMonitor:ep: 754,\tT: 1,539,103,\tG: 17,\tavg_r: 0.00929,\tavg_G: 16.8,\tt: 1830,\tdt: 2.837ms,\tSimpleTD/loss: 0.00466,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.0021\n",
      "INFO:TrainMonitor:ep: 755,\tT: 1,540,774,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.1,\tt: 1670,\tdt: 6.424ms,\tSimpleTD/loss: 0.00165,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0027\n",
      "INFO:TrainMonitor:ep: 756,\tT: 1,542,731,\tG: 16,\tavg_r: 0.00818,\tavg_G: 17,\tt: 1956,\tdt: 2.806ms,\tSimpleTD/loss: 0.00665,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00248\n",
      "INFO:TrainMonitor:ep: 757,\tT: 1,544,647,\tG: 14,\tavg_r: 0.00731,\tavg_G: 16.7,\tt: 1915,\tdt: 2.786ms,\tSimpleTD/loss: 0.00821,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00616\n",
      "INFO:TrainMonitor:ep: 758,\tT: 1,546,560,\tG: 16,\tavg_r: 0.00837,\tavg_G: 16.6,\tt: 1912,\tdt: 2.831ms,\tSimpleTD/loss: 0.00557,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00202\n",
      "INFO:TrainMonitor:ep: 759,\tT: 1,548,285,\tG: 19,\tavg_r: 0.011,\tavg_G: 16.9,\tt: 1724,\tdt: 2.773ms,\tSimpleTD/loss: 0.00223,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.0033\n",
      "INFO:TrainMonitor:ep: 760,\tT: 1,549,952,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.2,\tt: 1666,\tdt: 2.833ms,\tSimpleTD/loss: 0.00121,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00275\n",
      "INFO:TrainMonitor:ep: 761,\tT: 1,551,617,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.5,\tt: 1664,\tdt: 6.627ms,\tSimpleTD/loss: 0.0013,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00275\n",
      "INFO:TrainMonitor:ep: 762,\tT: 1,553,470,\tG: 17,\tavg_r: 0.00918,\tavg_G: 17.4,\tt: 1852,\tdt: 2.823ms,\tSimpleTD/loss: 0.00293,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00324\n",
      "INFO:TrainMonitor:ep: 763,\tT: 1,555,643,\tG: 11,\tavg_r: 0.00506,\tavg_G: 16.8,\tt: 2172,\tdt: 2.773ms,\tSimpleTD/loss: 0.01,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00451\n",
      "INFO:TrainMonitor:ep: 764,\tT: 1,557,396,\tG: 19,\tavg_r: 0.0108,\tavg_G: 17,\tt: 1752,\tdt: 2.812ms,\tSimpleTD/loss: 0.00278,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00292\n",
      "INFO:TrainMonitor:ep: 765,\tT: 1,559,329,\tG: 17,\tavg_r: 0.0088,\tavg_G: 17,\tt: 1932,\tdt: 2.763ms,\tSimpleTD/loss: 0.00474,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00372\n",
      "INFO:TrainMonitor:ep: 766,\tT: 1,561,001,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.3,\tt: 1671,\tdt: 6.160ms,\tSimpleTD/loss: 0.00127,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00307\n",
      "INFO:TrainMonitor:ep: 767,\tT: 1,562,687,\tG: 20,\tavg_r: 0.0119,\tavg_G: 17.6,\tt: 1685,\tdt: 2.842ms,\tSimpleTD/loss: 0.00209,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00302\n",
      "INFO:TrainMonitor:ep: 768,\tT: 1,564,502,\tG: 18,\tavg_r: 0.00992,\tavg_G: 17.6,\tt: 1814,\tdt: 2.799ms,\tSimpleTD/loss: 0.00502,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00181\n",
      "INFO:TrainMonitor:ep: 769,\tT: 1,566,312,\tG: 19,\tavg_r: 0.0105,\tavg_G: 17.7,\tt: 1809,\tdt: 2.786ms,\tSimpleTD/loss: 0.00579,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00519\n",
      "INFO:TrainMonitor:ep: 770,\tT: 1,568,276,\tG: 17,\tavg_r: 0.00866,\tavg_G: 17.7,\tt: 1963,\tdt: 2.770ms,\tSimpleTD/loss: 0.00315,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00289\n",
      "INFO:TrainMonitor:ep: 771,\tT: 1,570,020,\tG: 20,\tavg_r: 0.0115,\tavg_G: 17.9,\tt: 1743,\tdt: 6.385ms,\tSimpleTD/loss: 0.005,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00257\n",
      "INFO:TrainMonitor:ep: 772,\tT: 1,571,928,\tG: 17,\tavg_r: 0.00891,\tavg_G: 17.8,\tt: 1907,\tdt: 2.858ms,\tSimpleTD/loss: 0.00356,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00263\n",
      "INFO:TrainMonitor:ep: 773,\tT: 1,573,906,\tG: 16,\tavg_r: 0.00809,\tavg_G: 17.6,\tt: 1977,\tdt: 2.771ms,\tSimpleTD/loss: 0.00378,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00289\n",
      "INFO:TrainMonitor:ep: 774,\tT: 1,575,683,\tG: 18,\tavg_r: 0.0101,\tavg_G: 17.7,\tt: 1776,\tdt: 2.801ms,\tSimpleTD/loss: 0.00232,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00234\n",
      "INFO:TrainMonitor:ep: 775,\tT: 1,577,366,\tG: 20,\tavg_r: 0.0119,\tavg_G: 17.9,\tt: 1682,\tdt: 2.816ms,\tSimpleTD/loss: 0.00186,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00366\n",
      "INFO:TrainMonitor:ep: 776,\tT: 1,579,423,\tG: 12,\tavg_r: 0.00584,\tavg_G: 17.3,\tt: 2056,\tdt: 2.786ms,\tSimpleTD/loss: 0.00514,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00279\n",
      "INFO:TrainMonitor:ep: 777,\tT: 1,581,118,\tG: 20,\tavg_r: 0.0118,\tavg_G: 17.6,\tt: 1694,\tdt: 6.127ms,\tSimpleTD/loss: 0.00158,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0031\n",
      "INFO:TrainMonitor:ep: 778,\tT: 1,583,153,\tG: 14,\tavg_r: 0.00688,\tavg_G: 17.2,\tt: 2034,\tdt: 2.797ms,\tSimpleTD/loss: 0.00606,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00381\n",
      "INFO:TrainMonitor:ep: 779,\tT: 1,584,921,\tG: 19,\tavg_r: 0.0108,\tavg_G: 17.4,\tt: 1767,\tdt: 2.759ms,\tSimpleTD/loss: 0.00167,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00313\n",
      "INFO:TrainMonitor:ep: 780,\tT: 1,586,844,\tG: 16,\tavg_r: 0.00832,\tavg_G: 17.3,\tt: 1922,\tdt: 2.819ms,\tSimpleTD/loss: 0.00353,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00296\n",
      "INFO:TrainMonitor:ep: 781,\tT: 1,588,629,\tG: 19,\tavg_r: 0.0107,\tavg_G: 17.4,\tt: 1784,\tdt: 2.801ms,\tSimpleTD/loss: 0.00342,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00421\n",
      "INFO:TrainMonitor:ep: 782,\tT: 1,590,522,\tG: 16,\tavg_r: 0.00846,\tavg_G: 17.3,\tt: 1892,\tdt: 6.006ms,\tSimpleTD/loss: 0.00471,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00312\n",
      "INFO:TrainMonitor:ep: 783,\tT: 1,592,269,\tG: 19,\tavg_r: 0.0109,\tavg_G: 17.5,\tt: 1746,\tdt: 2.815ms,\tSimpleTD/loss: 0.00356,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00249\n",
      "INFO:TrainMonitor:ep: 784,\tT: 1,594,164,\tG: 18,\tavg_r: 0.0095,\tavg_G: 17.5,\tt: 1894,\tdt: 2.788ms,\tSimpleTD/loss: 0.00496,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.0033\n",
      "INFO:TrainMonitor:ep: 785,\tT: 1,595,888,\tG: 19,\tavg_r: 0.011,\tavg_G: 17.7,\tt: 1723,\tdt: 2.812ms,\tSimpleTD/loss: 0.00193,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00352\n",
      "INFO:TrainMonitor:ep: 786,\tT: 1,597,890,\tG: 14,\tavg_r: 0.007,\tavg_G: 17.3,\tt: 2001,\tdt: 2.801ms,\tSimpleTD/loss: 0.00422,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00112\n",
      "INFO:TrainMonitor:ep: 787,\tT: 1,599,814,\tG: 16,\tavg_r: 0.00832,\tavg_G: 17.2,\tt: 1923,\tdt: 2.814ms,\tSimpleTD/loss: 0.00467,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00351\n",
      "INFO:TrainMonitor:ep: 788,\tT: 1,602,005,\tG: 11,\tavg_r: 0.00502,\tavg_G: 16.6,\tt: 2190,\tdt: 6.426ms,\tSimpleTD/loss: 0.00709,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00449\n",
      "INFO:TrainMonitor:ep: 789,\tT: 1,603,686,\tG: 20,\tavg_r: 0.0119,\tavg_G: 16.9,\tt: 1680,\tdt: 2.813ms,\tSimpleTD/loss: 0.00176,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00221\n",
      "INFO:TrainMonitor:ep: 790,\tT: 1,605,371,\tG: 20,\tavg_r: 0.0119,\tavg_G: 17.2,\tt: 1684,\tdt: 2.782ms,\tSimpleTD/loss: 0.00129,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00248\n",
      "INFO:TrainMonitor:ep: 791,\tT: 1,607,764,\tG: 6,\tavg_r: 0.00251,\tavg_G: 16.1,\tt: 2392,\tdt: 2.808ms,\tSimpleTD/loss: 0.00781,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00373\n",
      "INFO:TrainMonitor:ep: 792,\tT: 1,609,608,\tG: 18,\tavg_r: 0.00977,\tavg_G: 16.3,\tt: 1843,\tdt: 2.781ms,\tSimpleTD/loss: 0.00472,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00441\n",
      "INFO:TrainMonitor:ep: 793,\tT: 1,611,389,\tG: 18,\tavg_r: 0.0101,\tavg_G: 16.4,\tt: 1780,\tdt: 5.909ms,\tSimpleTD/loss: 0.00441,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00292\n",
      "INFO:TrainMonitor:ep: 794,\tT: 1,613,267,\tG: 17,\tavg_r: 0.00906,\tavg_G: 16.5,\tt: 1877,\tdt: 2.789ms,\tSimpleTD/loss: 0.00416,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00246\n",
      "INFO:TrainMonitor:ep: 795,\tT: 1,615,007,\tG: 18,\tavg_r: 0.0104,\tavg_G: 16.7,\tt: 1739,\tdt: 2.815ms,\tSimpleTD/loss: 0.00265,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00226\n",
      "INFO:TrainMonitor:ep: 796,\tT: 1,616,819,\tG: 17,\tavg_r: 0.00939,\tavg_G: 16.7,\tt: 1811,\tdt: 2.810ms,\tSimpleTD/loss: 0.00535,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00361\n",
      "INFO:TrainMonitor:ep: 797,\tT: 1,618,696,\tG: 18,\tavg_r: 0.00959,\tavg_G: 16.8,\tt: 1876,\tdt: 2.804ms,\tSimpleTD/loss: 0.00529,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00528\n",
      "INFO:TrainMonitor:ep: 798,\tT: 1,620,462,\tG: 19,\tavg_r: 0.0108,\tavg_G: 17,\tt: 1765,\tdt: 6.506ms,\tSimpleTD/loss: 0.00329,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00194\n",
      "INFO:TrainMonitor:ep: 799,\tT: 1,622,218,\tG: 19,\tavg_r: 0.0108,\tavg_G: 17.2,\tt: 1755,\tdt: 2.814ms,\tSimpleTD/loss: 0.00211,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.0022\n",
      "INFO:TrainMonitor:ep: 800,\tT: 1,624,037,\tG: 18,\tavg_r: 0.0099,\tavg_G: 17.3,\tt: 1818,\tdt: 2.799ms,\tSimpleTD/loss: 0.00255,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00235\n",
      "INFO:TrainMonitor:ep: 801,\tT: 1,625,861,\tG: 17,\tavg_r: 0.00933,\tavg_G: 17.3,\tt: 1823,\tdt: 2.806ms,\tSimpleTD/loss: 0.00316,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00213\n",
      "INFO:TrainMonitor:ep: 802,\tT: 1,627,736,\tG: 15,\tavg_r: 0.008,\tavg_G: 17.1,\tt: 1874,\tdt: 2.843ms,\tSimpleTD/loss: 0.00586,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00263\n",
      "INFO:TrainMonitor:ep: 803,\tT: 1,629,679,\tG: 19,\tavg_r: 0.00978,\tavg_G: 17.2,\tt: 1942,\tdt: 2.772ms,\tSimpleTD/loss: 0.00426,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00398\n",
      "INFO:TrainMonitor:ep: 804,\tT: 1,631,391,\tG: 20,\tavg_r: 0.0117,\tavg_G: 17.5,\tt: 1711,\tdt: 7.063ms,\tSimpleTD/loss: 0.00217,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00253\n",
      "INFO:TrainMonitor:ep: 805,\tT: 1,633,302,\tG: 15,\tavg_r: 0.00785,\tavg_G: 17.3,\tt: 1910,\tdt: 2.775ms,\tSimpleTD/loss: 0.00817,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0032\n",
      "INFO:TrainMonitor:ep: 806,\tT: 1,634,971,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.5,\tt: 1668,\tdt: 2.838ms,\tSimpleTD/loss: 0.00267,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00388\n",
      "INFO:TrainMonitor:ep: 807,\tT: 1,636,636,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.8,\tt: 1664,\tdt: 2.770ms,\tSimpleTD/loss: 0.00112,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00269\n",
      "INFO:TrainMonitor:ep: 808,\tT: 1,638,782,\tG: 13,\tavg_r: 0.00606,\tavg_G: 17.3,\tt: 2145,\tdt: 2.820ms,\tSimpleTD/loss: 0.00756,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00435\n",
      "INFO:TrainMonitor:ep: 809,\tT: 1,640,887,\tG: 14,\tavg_r: 0.00665,\tavg_G: 17,\tt: 2104,\tdt: 5.723ms,\tSimpleTD/loss: 0.00796,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00575\n",
      "INFO:TrainMonitor:ep: 810,\tT: 1,642,762,\tG: 18,\tavg_r: 0.00961,\tavg_G: 17.1,\tt: 1874,\tdt: 2.783ms,\tSimpleTD/loss: 0.00316,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00357\n",
      "INFO:TrainMonitor:ep: 811,\tT: 1,644,435,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.4,\tt: 1672,\tdt: 2.828ms,\tSimpleTD/loss: 0.00308,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00325\n",
      "INFO:TrainMonitor:ep: 812,\tT: 1,646,271,\tG: 17,\tavg_r: 0.00926,\tavg_G: 17.3,\tt: 1835,\tdt: 2.780ms,\tSimpleTD/loss: 0.00545,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00291\n",
      "INFO:TrainMonitor:ep: 813,\tT: 1,648,087,\tG: 17,\tavg_r: 0.00937,\tavg_G: 17.3,\tt: 1815,\tdt: 2.793ms,\tSimpleTD/loss: 0.00496,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00259\n",
      "INFO:TrainMonitor:ep: 814,\tT: 1,649,784,\tG: 19,\tavg_r: 0.0112,\tavg_G: 17.5,\tt: 1696,\tdt: 2.816ms,\tSimpleTD/loss: 0.00168,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00206\n",
      "INFO:TrainMonitor:ep: 815,\tT: 1,651,451,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.7,\tt: 1666,\tdt: 6.000ms,\tSimpleTD/loss: 0.000895,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00356\n",
      "INFO:TrainMonitor:ep: 816,\tT: 1,653,120,\tG: 20,\tavg_r: 0.012,\tavg_G: 18,\tt: 1668,\tdt: 2.819ms,\tSimpleTD/loss: 0.000727,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00277\n",
      "INFO:TrainMonitor:ep: 817,\tT: 1,654,786,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.2,\tt: 1665,\tdt: 2.769ms,\tSimpleTD/loss: 0.000484,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00211\n",
      "INFO:TrainMonitor:ep: 818,\tT: 1,656,701,\tG: 17,\tavg_r: 0.00888,\tavg_G: 18,\tt: 1914,\tdt: 2.817ms,\tSimpleTD/loss: 0.00455,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00223\n",
      "INFO:TrainMonitor:ep: 819,\tT: 1,658,611,\tG: 15,\tavg_r: 0.00786,\tavg_G: 17.7,\tt: 1909,\tdt: 2.779ms,\tSimpleTD/loss: 0.00446,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.0021\n",
      "INFO:TrainMonitor:ep: 820,\tT: 1,660,427,\tG: 18,\tavg_r: 0.00992,\tavg_G: 17.8,\tt: 1815,\tdt: 6.956ms,\tSimpleTD/loss: 0.00373,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00408\n",
      "INFO:TrainMonitor:ep: 821,\tT: 1,662,469,\tG: 16,\tavg_r: 0.00784,\tavg_G: 17.6,\tt: 2041,\tdt: 2.799ms,\tSimpleTD/loss: 0.008,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00434\n",
      "INFO:TrainMonitor:ep: 822,\tT: 1,664,427,\tG: 16,\tavg_r: 0.00818,\tavg_G: 17.4,\tt: 1957,\tdt: 2.814ms,\tSimpleTD/loss: 0.00788,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00385\n",
      "INFO:TrainMonitor:ep: 823,\tT: 1,666,168,\tG: 19,\tavg_r: 0.0109,\tavg_G: 17.6,\tt: 1740,\tdt: 2.812ms,\tSimpleTD/loss: 0.00341,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00258\n",
      "INFO:TrainMonitor:ep: 824,\tT: 1,667,836,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.8,\tt: 1667,\tdt: 2.783ms,\tSimpleTD/loss: 0.000974,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00254\n",
      "INFO:TrainMonitor:ep: 825,\tT: 1,669,504,\tG: 20,\tavg_r: 0.012,\tavg_G: 18,\tt: 1667,\tdt: 2.816ms,\tSimpleTD/loss: 0.00255,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00299\n",
      "INFO:TrainMonitor:ep: 826,\tT: 1,671,524,\tG: 15,\tavg_r: 0.00743,\tavg_G: 17.7,\tt: 2019,\tdt: 5.632ms,\tSimpleTD/loss: 0.00625,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00373\n",
      "INFO:TrainMonitor:ep: 827,\tT: 1,673,253,\tG: 19,\tavg_r: 0.011,\tavg_G: 17.9,\tt: 1728,\tdt: 2.772ms,\tSimpleTD/loss: 0.00209,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00137\n",
      "INFO:TrainMonitor:ep: 828,\tT: 1,675,432,\tG: 13,\tavg_r: 0.00597,\tavg_G: 17.4,\tt: 2178,\tdt: 2.814ms,\tSimpleTD/loss: 0.00878,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00491\n",
      "INFO:TrainMonitor:ep: 829,\tT: 1,677,553,\tG: 12,\tavg_r: 0.00566,\tavg_G: 16.8,\tt: 2120,\tdt: 2.790ms,\tSimpleTD/loss: 0.00855,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00431\n",
      "INFO:TrainMonitor:ep: 830,\tT: 1,679,595,\tG: 16,\tavg_r: 0.00784,\tavg_G: 16.8,\tt: 2041,\tdt: 2.796ms,\tSimpleTD/loss: 0.00725,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00492\n",
      "INFO:TrainMonitor:ep: 831,\tT: 1,681,259,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.1,\tt: 1663,\tdt: 6.978ms,\tSimpleTD/loss: 0.00158,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00263\n",
      "INFO:TrainMonitor:ep: 832,\tT: 1,682,926,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.4,\tt: 1666,\tdt: 2.775ms,\tSimpleTD/loss: 0.000882,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00268\n",
      "INFO:TrainMonitor:ep: 833,\tT: 1,684,841,\tG: 16,\tavg_r: 0.00836,\tavg_G: 17.2,\tt: 1914,\tdt: 2.763ms,\tSimpleTD/loss: 0.00569,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00305\n",
      "INFO:TrainMonitor:ep: 834,\tT: 1,686,775,\tG: 16,\tavg_r: 0.00828,\tavg_G: 17.1,\tt: 1933,\tdt: 2.811ms,\tSimpleTD/loss: 0.00532,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00227\n",
      "INFO:TrainMonitor:ep: 835,\tT: 1,688,523,\tG: 19,\tavg_r: 0.0109,\tavg_G: 17.3,\tt: 1747,\tdt: 2.792ms,\tSimpleTD/loss: 0.00223,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00322\n",
      "INFO:TrainMonitor:ep: 836,\tT: 1,690,396,\tG: 18,\tavg_r: 0.00962,\tavg_G: 17.4,\tt: 1872,\tdt: 6.313ms,\tSimpleTD/loss: 0.00424,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00315\n",
      "INFO:TrainMonitor:ep: 837,\tT: 1,692,476,\tG: 15,\tavg_r: 0.00722,\tavg_G: 17.1,\tt: 2079,\tdt: 2.821ms,\tSimpleTD/loss: 0.00771,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00294\n",
      "INFO:TrainMonitor:ep: 838,\tT: 1,694,237,\tG: 19,\tavg_r: 0.0108,\tavg_G: 17.3,\tt: 1760,\tdt: 2.802ms,\tSimpleTD/loss: 0.00265,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0033\n",
      "INFO:TrainMonitor:ep: 839,\tT: 1,695,908,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.6,\tt: 1670,\tdt: 2.834ms,\tSimpleTD/loss: 0.00117,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00277\n",
      "INFO:TrainMonitor:ep: 840,\tT: 1,697,760,\tG: 17,\tavg_r: 0.00918,\tavg_G: 17.5,\tt: 1851,\tdt: 2.801ms,\tSimpleTD/loss: 0.005,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00114\n",
      "INFO:TrainMonitor:ep: 841,\tT: 1,699,808,\tG: 12,\tavg_r: 0.00586,\tavg_G: 17,\tt: 2047,\tdt: 2.805ms,\tSimpleTD/loss: 0.00767,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00529\n",
      "INFO:TrainMonitor:ep: 842,\tT: 1,701,918,\tG: 11,\tavg_r: 0.00522,\tavg_G: 16.4,\tt: 2109,\tdt: 6.117ms,\tSimpleTD/loss: 0.0133,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00558\n",
      "INFO:TrainMonitor:ep: 843,\tT: 1,703,585,\tG: 20,\tavg_r: 0.012,\tavg_G: 16.7,\tt: 1666,\tdt: 2.822ms,\tSimpleTD/loss: 0.00256,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00334\n",
      "INFO:TrainMonitor:ep: 844,\tT: 1,705,371,\tG: 18,\tavg_r: 0.0101,\tavg_G: 16.9,\tt: 1785,\tdt: 2.819ms,\tSimpleTD/loss: 0.00551,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00241\n",
      "INFO:TrainMonitor:ep: 845,\tT: 1,707,036,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.2,\tt: 1664,\tdt: 2.772ms,\tSimpleTD/loss: 0.00122,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00318\n",
      "INFO:TrainMonitor:ep: 846,\tT: 1,708,982,\tG: 17,\tavg_r: 0.00874,\tavg_G: 17.2,\tt: 1945,\tdt: 2.812ms,\tSimpleTD/loss: 0.00402,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00321\n",
      "INFO:TrainMonitor:ep: 847,\tT: 1,710,649,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.4,\tt: 1666,\tdt: 6.449ms,\tSimpleTD/loss: 0.000821,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00296\n",
      "INFO:TrainMonitor:ep: 848,\tT: 1,712,320,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.7,\tt: 1670,\tdt: 2.819ms,\tSimpleTD/loss: 0.00127,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00245\n",
      "INFO:TrainMonitor:ep: 849,\tT: 1,713,988,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.9,\tt: 1667,\tdt: 2.773ms,\tSimpleTD/loss: 0.000671,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.0021\n",
      "INFO:TrainMonitor:ep: 850,\tT: 1,715,946,\tG: 15,\tavg_r: 0.00766,\tavg_G: 17.6,\tt: 1957,\tdt: 2.815ms,\tSimpleTD/loss: 0.00533,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00312\n",
      "INFO:TrainMonitor:ep: 851,\tT: 1,717,673,\tG: 20,\tavg_r: 0.0116,\tavg_G: 17.9,\tt: 1726,\tdt: 2.807ms,\tSimpleTD/loss: 0.00169,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00378\n",
      "INFO:TrainMonitor:ep: 852,\tT: 1,719,758,\tG: 13,\tavg_r: 0.00624,\tavg_G: 17.4,\tt: 2084,\tdt: 2.790ms,\tSimpleTD/loss: 0.0053,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00183\n",
      "INFO:TrainMonitor:ep: 853,\tT: 1,721,626,\tG: 17,\tavg_r: 0.00911,\tavg_G: 17.3,\tt: 1867,\tdt: 5.898ms,\tSimpleTD/loss: 0.00419,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00308\n",
      "INFO:TrainMonitor:ep: 854,\tT: 1,723,291,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.6,\tt: 1664,\tdt: 2.841ms,\tSimpleTD/loss: 0.00139,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00321\n",
      "INFO:TrainMonitor:ep: 855,\tT: 1,724,958,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.9,\tt: 1666,\tdt: 2.765ms,\tSimpleTD/loss: 0.00106,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0029\n",
      "INFO:TrainMonitor:ep: 856,\tT: 1,726,633,\tG: 20,\tavg_r: 0.0119,\tavg_G: 18.1,\tt: 1674,\tdt: 2.835ms,\tSimpleTD/loss: 0.00144,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00322\n",
      "INFO:TrainMonitor:ep: 857,\tT: 1,728,298,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.3,\tt: 1664,\tdt: 2.766ms,\tSimpleTD/loss: 0.000584,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00226\n",
      "INFO:TrainMonitor:ep: 858,\tT: 1,730,097,\tG: 18,\tavg_r: 0.01,\tavg_G: 18.2,\tt: 1798,\tdt: 5.924ms,\tSimpleTD/loss: 0.00335,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00298\n",
      "INFO:TrainMonitor:ep: 859,\tT: 1,731,858,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18.3,\tt: 1760,\tdt: 2.814ms,\tSimpleTD/loss: 0.00233,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00282\n",
      "INFO:TrainMonitor:ep: 860,\tT: 1,733,698,\tG: 18,\tavg_r: 0.00979,\tavg_G: 18.3,\tt: 1839,\tdt: 2.791ms,\tSimpleTD/loss: 0.00172,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00218\n",
      "INFO:TrainMonitor:ep: 861,\tT: 1,735,444,\tG: 19,\tavg_r: 0.0109,\tavg_G: 18.4,\tt: 1745,\tdt: 2.804ms,\tSimpleTD/loss: 0.00307,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00425\n",
      "INFO:TrainMonitor:ep: 862,\tT: 1,737,369,\tG: 17,\tavg_r: 0.00884,\tavg_G: 18.2,\tt: 1924,\tdt: 2.809ms,\tSimpleTD/loss: 0.00509,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00345\n",
      "INFO:TrainMonitor:ep: 863,\tT: 1,739,121,\tG: 19,\tavg_r: 0.0109,\tavg_G: 18.3,\tt: 1751,\tdt: 2.751ms,\tSimpleTD/loss: 0.00312,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00391\n",
      "INFO:TrainMonitor:ep: 864,\tT: 1,740,915,\tG: 19,\tavg_r: 0.0106,\tavg_G: 18.4,\tt: 1793,\tdt: 6.198ms,\tSimpleTD/loss: 0.00248,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.002\n",
      "INFO:TrainMonitor:ep: 865,\tT: 1,742,757,\tG: 18,\tavg_r: 0.00978,\tavg_G: 18.3,\tt: 1841,\tdt: 2.874ms,\tSimpleTD/loss: 0.00336,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00241\n",
      "INFO:TrainMonitor:ep: 866,\tT: 1,744,586,\tG: 18,\tavg_r: 0.00985,\tavg_G: 18.3,\tt: 1828,\tdt: 2.804ms,\tSimpleTD/loss: 0.00307,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00266\n",
      "INFO:TrainMonitor:ep: 867,\tT: 1,746,816,\tG: 12,\tavg_r: 0.00538,\tavg_G: 17.7,\tt: 2229,\tdt: 2.782ms,\tSimpleTD/loss: 0.0064,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.0048\n",
      "INFO:TrainMonitor:ep: 868,\tT: 1,748,546,\tG: 20,\tavg_r: 0.0116,\tavg_G: 17.9,\tt: 1729,\tdt: 2.817ms,\tSimpleTD/loss: 0.00359,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00375\n",
      "INFO:TrainMonitor:ep: 869,\tT: 1,750,475,\tG: 18,\tavg_r: 0.00934,\tavg_G: 17.9,\tt: 1928,\tdt: 5.689ms,\tSimpleTD/loss: 0.00463,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00229\n",
      "INFO:TrainMonitor:ep: 870,\tT: 1,752,247,\tG: 19,\tavg_r: 0.0107,\tavg_G: 18,\tt: 1771,\tdt: 2.819ms,\tSimpleTD/loss: 0.00279,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00286\n",
      "INFO:TrainMonitor:ep: 871,\tT: 1,753,934,\tG: 20,\tavg_r: 0.0119,\tavg_G: 18.2,\tt: 1686,\tdt: 2.776ms,\tSimpleTD/loss: 0.0026,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00438\n",
      "INFO:TrainMonitor:ep: 872,\tT: 1,755,732,\tG: 18,\tavg_r: 0.01,\tavg_G: 18.2,\tt: 1797,\tdt: 2.797ms,\tSimpleTD/loss: 0.00435,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00373\n",
      "INFO:TrainMonitor:ep: 873,\tT: 1,757,529,\tG: 18,\tavg_r: 0.01,\tavg_G: 18.2,\tt: 1796,\tdt: 2.806ms,\tSimpleTD/loss: 0.00222,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00186\n",
      "INFO:TrainMonitor:ep: 874,\tT: 1,759,256,\tG: 19,\tavg_r: 0.011,\tavg_G: 18.3,\tt: 1726,\tdt: 2.812ms,\tSimpleTD/loss: 0.00404,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00337\n",
      "INFO:TrainMonitor:ep: 875,\tT: 1,760,920,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.4,\tt: 1663,\tdt: 6.575ms,\tSimpleTD/loss: 0.001,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00241\n",
      "INFO:TrainMonitor:ep: 876,\tT: 1,762,676,\tG: 18,\tavg_r: 0.0103,\tavg_G: 18.4,\tt: 1755,\tdt: 2.779ms,\tSimpleTD/loss: 0.00335,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.0022\n",
      "INFO:TrainMonitor:ep: 877,\tT: 1,764,340,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.6,\tt: 1663,\tdt: 2.858ms,\tSimpleTD/loss: 0.000733,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00258\n",
      "INFO:TrainMonitor:ep: 878,\tT: 1,766,176,\tG: 17,\tavg_r: 0.00926,\tavg_G: 18.4,\tt: 1835,\tdt: 2.786ms,\tSimpleTD/loss: 0.00388,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.0023\n",
      "INFO:TrainMonitor:ep: 879,\tT: 1,768,211,\tG: 16,\tavg_r: 0.00787,\tavg_G: 18.2,\tt: 2034,\tdt: 2.804ms,\tSimpleTD/loss: 0.00437,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.0034\n",
      "INFO:TrainMonitor:ep: 880,\tT: 1,769,966,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18.2,\tt: 1754,\tdt: 2.798ms,\tSimpleTD/loss: 0.0024,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00309\n",
      "INFO:TrainMonitor:ep: 881,\tT: 1,771,693,\tG: 19,\tavg_r: 0.011,\tavg_G: 18.3,\tt: 1726,\tdt: 6.006ms,\tSimpleTD/loss: 0.00316,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.0031\n",
      "INFO:TrainMonitor:ep: 882,\tT: 1,773,568,\tG: 16,\tavg_r: 0.00854,\tavg_G: 18.1,\tt: 1874,\tdt: 2.773ms,\tSimpleTD/loss: 0.00466,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00483\n",
      "INFO:TrainMonitor:ep: 883,\tT: 1,775,282,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.2,\tt: 1713,\tdt: 2.796ms,\tSimpleTD/loss: 0.00231,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0019\n",
      "INFO:TrainMonitor:ep: 884,\tT: 1,777,026,\tG: 19,\tavg_r: 0.0109,\tavg_G: 18.3,\tt: 1743,\tdt: 2.765ms,\tSimpleTD/loss: 0.0023,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00346\n",
      "INFO:TrainMonitor:ep: 885,\tT: 1,778,694,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.4,\tt: 1667,\tdt: 2.821ms,\tSimpleTD/loss: 0.00213,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00332\n",
      "INFO:TrainMonitor:ep: 886,\tT: 1,780,352,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.6,\tt: 1657,\tdt: 6.156ms,\tSimpleTD/loss: 0.000979,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0022\n",
      "INFO:TrainMonitor:ep: 887,\tT: 1,782,010,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.7,\tt: 1657,\tdt: 2.840ms,\tSimpleTD/loss: 0.000576,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00248\n",
      "INFO:TrainMonitor:ep: 888,\tT: 1,783,756,\tG: 19,\tavg_r: 0.0109,\tavg_G: 18.8,\tt: 1745,\tdt: 2.805ms,\tSimpleTD/loss: 0.00146,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00234\n",
      "INFO:TrainMonitor:ep: 889,\tT: 1,785,543,\tG: 19,\tavg_r: 0.0106,\tavg_G: 18.8,\tt: 1786,\tdt: 2.831ms,\tSimpleTD/loss: 0.00392,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00406\n",
      "INFO:TrainMonitor:ep: 890,\tT: 1,787,287,\tG: 20,\tavg_r: 0.0115,\tavg_G: 18.9,\tt: 1743,\tdt: 2.758ms,\tSimpleTD/loss: 0.00181,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00308\n",
      "INFO:TrainMonitor:ep: 891,\tT: 1,789,029,\tG: 19,\tavg_r: 0.0109,\tavg_G: 18.9,\tt: 1741,\tdt: 2.829ms,\tSimpleTD/loss: 0.000855,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00231\n",
      "INFO:TrainMonitor:ep: 892,\tT: 1,790,711,\tG: 20,\tavg_r: 0.0119,\tavg_G: 19,\tt: 1681,\tdt: 6.361ms,\tSimpleTD/loss: 0.00122,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00244\n",
      "INFO:TrainMonitor:ep: 893,\tT: 1,792,917,\tG: 12,\tavg_r: 0.00544,\tavg_G: 18.3,\tt: 2205,\tdt: 2.783ms,\tSimpleTD/loss: 0.00748,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00493\n",
      "INFO:TrainMonitor:ep: 894,\tT: 1,794,932,\tG: 17,\tavg_r: 0.00844,\tavg_G: 18.2,\tt: 2014,\tdt: 2.823ms,\tSimpleTD/loss: 0.00547,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00303\n",
      "INFO:TrainMonitor:ep: 895,\tT: 1,796,889,\tG: 16,\tavg_r: 0.00818,\tavg_G: 18,\tt: 1956,\tdt: 2.846ms,\tSimpleTD/loss: 0.00693,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00293\n",
      "INFO:TrainMonitor:ep: 896,\tT: 1,798,928,\tG: 16,\tavg_r: 0.00785,\tavg_G: 17.8,\tt: 2038,\tdt: 2.817ms,\tSimpleTD/loss: 0.00582,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00389\n",
      "INFO:TrainMonitor:ep: 897,\tT: 1,800,932,\tG: 17,\tavg_r: 0.00849,\tavg_G: 17.7,\tt: 2003,\tdt: 5.870ms,\tSimpleTD/loss: 0.00513,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00364\n",
      "INFO:TrainMonitor:ep: 898,\tT: 1,802,815,\tG: 16,\tavg_r: 0.0085,\tavg_G: 17.5,\tt: 1882,\tdt: 2.794ms,\tSimpleTD/loss: 0.00535,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00426\n",
      "INFO:TrainMonitor:ep: 899,\tT: 1,804,763,\tG: 16,\tavg_r: 0.00822,\tavg_G: 17.4,\tt: 1947,\tdt: 2.849ms,\tSimpleTD/loss: 0.00419,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00291\n",
      "INFO:TrainMonitor:ep: 900,\tT: 1,806,740,\tG: 14,\tavg_r: 0.00709,\tavg_G: 17,\tt: 1976,\tdt: 2.772ms,\tSimpleTD/loss: 0.00938,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00112\n",
      "INFO:TrainMonitor:ep: 901,\tT: 1,808,529,\tG: 19,\tavg_r: 0.0106,\tavg_G: 17.2,\tt: 1788,\tdt: 2.814ms,\tSimpleTD/loss: 0.00418,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00362\n",
      "INFO:TrainMonitor:ep: 902,\tT: 1,810,282,\tG: 18,\tavg_r: 0.0103,\tavg_G: 17.3,\tt: 1752,\tdt: 6.085ms,\tSimpleTD/loss: 0.00256,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00255\n",
      "INFO:TrainMonitor:ep: 903,\tT: 1,812,214,\tG: 17,\tavg_r: 0.0088,\tavg_G: 17.3,\tt: 1931,\tdt: 2.827ms,\tSimpleTD/loss: 0.00228,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.0017\n",
      "INFO:TrainMonitor:ep: 904,\tT: 1,814,162,\tG: 17,\tavg_r: 0.00873,\tavg_G: 17.3,\tt: 1947,\tdt: 2.788ms,\tSimpleTD/loss: 0.00513,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00326\n",
      "INFO:TrainMonitor:ep: 905,\tT: 1,815,876,\tG: 19,\tavg_r: 0.0111,\tavg_G: 17.4,\tt: 1713,\tdt: 2.846ms,\tSimpleTD/loss: 0.00481,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00175\n",
      "INFO:TrainMonitor:ep: 906,\tT: 1,817,815,\tG: 16,\tavg_r: 0.00826,\tavg_G: 17.3,\tt: 1938,\tdt: 2.839ms,\tSimpleTD/loss: 0.00641,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: 0.000582\n",
      "INFO:TrainMonitor:ep: 907,\tT: 1,819,633,\tG: 17,\tavg_r: 0.00936,\tavg_G: 17.3,\tt: 1817,\tdt: 2.822ms,\tSimpleTD/loss: 0.00555,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00343\n",
      "INFO:TrainMonitor:ep: 908,\tT: 1,821,545,\tG: 18,\tavg_r: 0.00942,\tavg_G: 17.3,\tt: 1911,\tdt: 5.598ms,\tSimpleTD/loss: 0.00373,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00236\n",
      "INFO:TrainMonitor:ep: 909,\tT: 1,823,337,\tG: 18,\tavg_r: 0.0101,\tavg_G: 17.4,\tt: 1791,\tdt: 2.805ms,\tSimpleTD/loss: 0.0029,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00278\n",
      "INFO:TrainMonitor:ep: 910,\tT: 1,825,198,\tG: 17,\tavg_r: 0.00914,\tavg_G: 17.4,\tt: 1860,\tdt: 2.789ms,\tSimpleTD/loss: 0.00404,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00264\n",
      "INFO:TrainMonitor:ep: 911,\tT: 1,827,007,\tG: 17,\tavg_r: 0.0094,\tavg_G: 17.3,\tt: 1808,\tdt: 2.851ms,\tSimpleTD/loss: 0.00559,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00276\n",
      "INFO:TrainMonitor:ep: 912,\tT: 1,828,664,\tG: 20,\tavg_r: 0.0121,\tavg_G: 17.6,\tt: 1656,\tdt: 2.779ms,\tSimpleTD/loss: 0.00112,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00345\n",
      "INFO:TrainMonitor:ep: 913,\tT: 1,830,451,\tG: 18,\tavg_r: 0.0101,\tavg_G: 17.6,\tt: 1786,\tdt: 6.217ms,\tSimpleTD/loss: 0.00513,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00189\n",
      "INFO:TrainMonitor:ep: 914,\tT: 1,832,167,\tG: 19,\tavg_r: 0.0111,\tavg_G: 17.8,\tt: 1715,\tdt: 2.834ms,\tSimpleTD/loss: 0.00224,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00164\n",
      "INFO:TrainMonitor:ep: 915,\tT: 1,833,888,\tG: 19,\tavg_r: 0.011,\tavg_G: 17.9,\tt: 1720,\tdt: 2.768ms,\tSimpleTD/loss: 0.00181,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00245\n",
      "INFO:TrainMonitor:ep: 916,\tT: 1,835,661,\tG: 18,\tavg_r: 0.0102,\tavg_G: 17.9,\tt: 1772,\tdt: 2.824ms,\tSimpleTD/loss: 0.00455,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00315\n",
      "INFO:TrainMonitor:ep: 917,\tT: 1,837,327,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.1,\tt: 1665,\tdt: 2.825ms,\tSimpleTD/loss: 0.00113,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00247\n",
      "INFO:TrainMonitor:ep: 918,\tT: 1,839,030,\tG: 19,\tavg_r: 0.0112,\tavg_G: 18.2,\tt: 1702,\tdt: 2.770ms,\tSimpleTD/loss: 0.00281,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.0033\n",
      "INFO:TrainMonitor:ep: 919,\tT: 1,840,794,\tG: 18,\tavg_r: 0.0102,\tavg_G: 18.2,\tt: 1763,\tdt: 5.767ms,\tSimpleTD/loss: 0.00535,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00379\n",
      "INFO:TrainMonitor:ep: 920,\tT: 1,842,543,\tG: 18,\tavg_r: 0.0103,\tavg_G: 18.2,\tt: 1748,\tdt: 2.811ms,\tSimpleTD/loss: 0.00282,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00225\n",
      "INFO:TrainMonitor:ep: 921,\tT: 1,844,243,\tG: 20,\tavg_r: 0.0118,\tavg_G: 18.3,\tt: 1699,\tdt: 2.824ms,\tSimpleTD/loss: 0.00246,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0024\n",
      "INFO:TrainMonitor:ep: 922,\tT: 1,845,969,\tG: 19,\tavg_r: 0.011,\tavg_G: 18.4,\tt: 1725,\tdt: 2.812ms,\tSimpleTD/loss: 0.00174,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.0025\n",
      "INFO:TrainMonitor:ep: 923,\tT: 1,847,613,\tG: 21,\tavg_r: 0.0128,\tavg_G: 18.7,\tt: 1643,\tdt: 2.767ms,\tSimpleTD/loss: 0.00115,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00206\n",
      "INFO:TrainMonitor:ep: 924,\tT: 1,849,276,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.8,\tt: 1662,\tdt: 2.781ms,\tSimpleTD/loss: 0.00137,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00293\n",
      "INFO:TrainMonitor:ep: 925,\tT: 1,851,402,\tG: 12,\tavg_r: 0.00565,\tavg_G: 18.1,\tt: 2125,\tdt: 5.470ms,\tSimpleTD/loss: 0.00882,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00389\n",
      "INFO:TrainMonitor:ep: 926,\tT: 1,853,121,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.2,\tt: 1718,\tdt: 2.760ms,\tSimpleTD/loss: 0.00446,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00233\n",
      "INFO:TrainMonitor:ep: 927,\tT: 1,854,885,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18.3,\tt: 1763,\tdt: 2.813ms,\tSimpleTD/loss: 0.00222,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00234\n",
      "INFO:TrainMonitor:ep: 928,\tT: 1,856,788,\tG: 14,\tavg_r: 0.00736,\tavg_G: 17.9,\tt: 1902,\tdt: 2.827ms,\tSimpleTD/loss: 0.00721,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.0041\n",
      "INFO:TrainMonitor:ep: 929,\tT: 1,858,445,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.1,\tt: 1656,\tdt: 2.791ms,\tSimpleTD/loss: 0.00174,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00341\n",
      "INFO:TrainMonitor:ep: 930,\tT: 1,860,110,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.3,\tt: 1664,\tdt: 6.058ms,\tSimpleTD/loss: 0.00104,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00243\n",
      "INFO:TrainMonitor:ep: 931,\tT: 1,861,861,\tG: 18,\tavg_r: 0.0103,\tavg_G: 18.2,\tt: 1750,\tdt: 2.802ms,\tSimpleTD/loss: 0.0024,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00263\n",
      "INFO:TrainMonitor:ep: 932,\tT: 1,863,527,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.4,\tt: 1665,\tdt: 2.764ms,\tSimpleTD/loss: 0.000915,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00218\n",
      "INFO:TrainMonitor:ep: 933,\tT: 1,865,316,\tG: 20,\tavg_r: 0.0112,\tavg_G: 18.6,\tt: 1788,\tdt: 2.801ms,\tSimpleTD/loss: 0.00236,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00267\n",
      "INFO:TrainMonitor:ep: 934,\tT: 1,866,987,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.7,\tt: 1670,\tdt: 2.816ms,\tSimpleTD/loss: 0.00178,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00277\n",
      "INFO:TrainMonitor:ep: 935,\tT: 1,868,884,\tG: 19,\tavg_r: 0.01,\tavg_G: 18.7,\tt: 1896,\tdt: 2.783ms,\tSimpleTD/loss: 0.00334,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.004\n",
      "INFO:TrainMonitor:ep: 936,\tT: 1,870,700,\tG: 19,\tavg_r: 0.0105,\tavg_G: 18.8,\tt: 1815,\tdt: 5.723ms,\tSimpleTD/loss: 0.00304,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00285\n",
      "INFO:TrainMonitor:ep: 937,\tT: 1,872,416,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.8,\tt: 1715,\tdt: 2.819ms,\tSimpleTD/loss: 0.00217,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00176\n",
      "INFO:TrainMonitor:ep: 938,\tT: 1,874,340,\tG: 17,\tavg_r: 0.00884,\tavg_G: 18.6,\tt: 1923,\tdt: 2.774ms,\tSimpleTD/loss: 0.00439,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00389\n",
      "INFO:TrainMonitor:ep: 939,\tT: 1,876,052,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.7,\tt: 1711,\tdt: 2.854ms,\tSimpleTD/loss: 0.00256,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00338\n",
      "INFO:TrainMonitor:ep: 940,\tT: 1,877,720,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.8,\tt: 1667,\tdt: 2.777ms,\tSimpleTD/loss: 0.000944,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.0029\n",
      "INFO:TrainMonitor:ep: 941,\tT: 1,879,442,\tG: 19,\tavg_r: 0.011,\tavg_G: 18.8,\tt: 1721,\tdt: 2.818ms,\tSimpleTD/loss: 0.00159,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00246\n",
      "INFO:TrainMonitor:ep: 942,\tT: 1,881,113,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.9,\tt: 1670,\tdt: 6.313ms,\tSimpleTD/loss: 0.000883,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00235\n",
      "INFO:TrainMonitor:ep: 943,\tT: 1,882,801,\tG: 21,\tavg_r: 0.0124,\tavg_G: 19.1,\tt: 1687,\tdt: 2.773ms,\tSimpleTD/loss: 0.00133,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.0022\n",
      "INFO:TrainMonitor:ep: 944,\tT: 1,884,665,\tG: 17,\tavg_r: 0.00913,\tavg_G: 18.9,\tt: 1863,\tdt: 2.846ms,\tSimpleTD/loss: 0.00455,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00198\n",
      "INFO:TrainMonitor:ep: 945,\tT: 1,886,519,\tG: 18,\tavg_r: 0.00971,\tavg_G: 18.8,\tt: 1853,\tdt: 2.792ms,\tSimpleTD/loss: 0.0038,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.0025\n",
      "INFO:TrainMonitor:ep: 946,\tT: 1,888,277,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18.8,\tt: 1757,\tdt: 2.802ms,\tSimpleTD/loss: 0.00188,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00317\n",
      "INFO:TrainMonitor:ep: 947,\tT: 1,890,931,\tG: 7,\tavg_r: 0.00264,\tavg_G: 17.7,\tt: 2653,\tdt: 5.183ms,\tSimpleTD/loss: 0.0109,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00663\n",
      "INFO:TrainMonitor:ep: 948,\tT: 1,892,877,\tG: 16,\tavg_r: 0.00823,\tavg_G: 17.5,\tt: 1945,\tdt: 2.822ms,\tSimpleTD/loss: 0.00716,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00255\n",
      "INFO:TrainMonitor:ep: 949,\tT: 1,894,806,\tG: 16,\tavg_r: 0.0083,\tavg_G: 17.3,\tt: 1928,\tdt: 2.774ms,\tSimpleTD/loss: 0.00655,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00328\n",
      "INFO:TrainMonitor:ep: 950,\tT: 1,896,802,\tG: 17,\tavg_r: 0.00852,\tavg_G: 17.3,\tt: 1995,\tdt: 2.819ms,\tSimpleTD/loss: 0.00611,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00322\n",
      "INFO:TrainMonitor:ep: 951,\tT: 1,898,499,\tG: 19,\tavg_r: 0.0112,\tavg_G: 17.5,\tt: 1696,\tdt: 2.820ms,\tSimpleTD/loss: 0.00196,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00192\n",
      "INFO:TrainMonitor:ep: 952,\tT: 1,900,345,\tG: 17,\tavg_r: 0.00921,\tavg_G: 17.4,\tt: 1845,\tdt: 5.959ms,\tSimpleTD/loss: 0.00472,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00351\n",
      "INFO:TrainMonitor:ep: 953,\tT: 1,902,094,\tG: 18,\tavg_r: 0.0103,\tavg_G: 17.5,\tt: 1748,\tdt: 2.803ms,\tSimpleTD/loss: 0.00361,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00258\n",
      "INFO:TrainMonitor:ep: 954,\tT: 1,903,961,\tG: 17,\tavg_r: 0.00911,\tavg_G: 17.4,\tt: 1866,\tdt: 2.784ms,\tSimpleTD/loss: 0.00374,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00234\n",
      "INFO:TrainMonitor:ep: 955,\tT: 1,905,925,\tG: 15,\tavg_r: 0.00764,\tavg_G: 17.2,\tt: 1963,\tdt: 2.810ms,\tSimpleTD/loss: 0.00689,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00282\n",
      "INFO:TrainMonitor:ep: 956,\tT: 1,907,757,\tG: 17,\tavg_r: 0.00928,\tavg_G: 17.2,\tt: 1831,\tdt: 2.827ms,\tSimpleTD/loss: 0.00484,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00426\n",
      "INFO:TrainMonitor:ep: 957,\tT: 1,909,658,\tG: 17,\tavg_r: 0.00895,\tavg_G: 17.2,\tt: 1900,\tdt: 2.795ms,\tSimpleTD/loss: 0.00391,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00138\n",
      "INFO:TrainMonitor:ep: 958,\tT: 1,911,359,\tG: 20,\tavg_r: 0.0118,\tavg_G: 17.4,\tt: 1700,\tdt: 6.155ms,\tSimpleTD/loss: 0.0018,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00336\n",
      "INFO:TrainMonitor:ep: 959,\tT: 1,913,128,\tG: 18,\tavg_r: 0.0102,\tavg_G: 17.5,\tt: 1768,\tdt: 2.821ms,\tSimpleTD/loss: 0.00342,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00204\n",
      "INFO:TrainMonitor:ep: 960,\tT: 1,914,841,\tG: 19,\tavg_r: 0.0111,\tavg_G: 17.6,\tt: 1712,\tdt: 2.782ms,\tSimpleTD/loss: 0.00222,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00293\n",
      "INFO:TrainMonitor:ep: 961,\tT: 1,916,708,\tG: 17,\tavg_r: 0.00911,\tavg_G: 17.6,\tt: 1866,\tdt: 2.842ms,\tSimpleTD/loss: 0.00334,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00303\n",
      "INFO:TrainMonitor:ep: 962,\tT: 1,918,503,\tG: 19,\tavg_r: 0.0106,\tavg_G: 17.7,\tt: 1794,\tdt: 2.813ms,\tSimpleTD/loss: 0.00299,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00232\n",
      "INFO:TrainMonitor:ep: 963,\tT: 1,920,172,\tG: 20,\tavg_r: 0.012,\tavg_G: 18,\tt: 1668,\tdt: 6.304ms,\tSimpleTD/loss: 0.00144,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00258\n",
      "INFO:TrainMonitor:ep: 964,\tT: 1,921,990,\tG: 18,\tavg_r: 0.00991,\tavg_G: 18,\tt: 1817,\tdt: 2.810ms,\tSimpleTD/loss: 0.00313,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00212\n",
      "INFO:TrainMonitor:ep: 965,\tT: 1,923,740,\tG: 19,\tavg_r: 0.0109,\tavg_G: 18.1,\tt: 1749,\tdt: 2.817ms,\tSimpleTD/loss: 0.0028,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.0034\n",
      "INFO:TrainMonitor:ep: 966,\tT: 1,925,398,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.3,\tt: 1657,\tdt: 2.831ms,\tSimpleTD/loss: 0.00132,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00251\n",
      "INFO:TrainMonitor:ep: 967,\tT: 1,927,067,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.4,\tt: 1668,\tdt: 2.781ms,\tSimpleTD/loss: 0.000799,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00226\n",
      "INFO:TrainMonitor:ep: 968,\tT: 1,928,823,\tG: 18,\tavg_r: 0.0103,\tavg_G: 18.4,\tt: 1755,\tdt: 2.804ms,\tSimpleTD/loss: 0.0052,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00149\n",
      "INFO:TrainMonitor:ep: 969,\tT: 1,930,486,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.5,\tt: 1662,\tdt: 5.918ms,\tSimpleTD/loss: 0.00118,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00317\n",
      "INFO:TrainMonitor:ep: 970,\tT: 1,932,510,\tG: 14,\tavg_r: 0.00692,\tavg_G: 18.1,\tt: 2023,\tdt: 2.814ms,\tSimpleTD/loss: 0.00571,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00241\n",
      "INFO:TrainMonitor:ep: 971,\tT: 1,934,176,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.3,\tt: 1665,\tdt: 2.839ms,\tSimpleTD/loss: 0.000964,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00314\n",
      "INFO:TrainMonitor:ep: 972,\tT: 1,935,885,\tG: 20,\tavg_r: 0.0117,\tavg_G: 18.5,\tt: 1708,\tdt: 2.833ms,\tSimpleTD/loss: 0.00134,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00201\n",
      "INFO:TrainMonitor:ep: 973,\tT: 1,937,754,\tG: 19,\tavg_r: 0.0102,\tavg_G: 18.5,\tt: 1868,\tdt: 2.819ms,\tSimpleTD/loss: 0.00393,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00406\n",
      "INFO:TrainMonitor:ep: 974,\tT: 1,939,472,\tG: 20,\tavg_r: 0.0116,\tavg_G: 18.7,\tt: 1717,\tdt: 2.840ms,\tSimpleTD/loss: 0.00104,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00261\n",
      "INFO:TrainMonitor:ep: 975,\tT: 1,941,135,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.8,\tt: 1662,\tdt: 6.227ms,\tSimpleTD/loss: 0.000569,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00283\n",
      "INFO:TrainMonitor:ep: 976,\tT: 1,942,997,\tG: 17,\tavg_r: 0.00913,\tavg_G: 18.6,\tt: 1861,\tdt: 2.793ms,\tSimpleTD/loss: 0.00247,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.000897\n",
      "INFO:TrainMonitor:ep: 977,\tT: 1,944,930,\tG: 16,\tavg_r: 0.00828,\tavg_G: 18.4,\tt: 1932,\tdt: 2.829ms,\tSimpleTD/loss: 0.00428,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00284\n",
      "INFO:TrainMonitor:ep: 978,\tT: 1,946,813,\tG: 18,\tavg_r: 0.00956,\tavg_G: 18.3,\tt: 1882,\tdt: 2.806ms,\tSimpleTD/loss: 0.00537,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00218\n",
      "INFO:TrainMonitor:ep: 979,\tT: 1,948,625,\tG: 17,\tavg_r: 0.00939,\tavg_G: 18.2,\tt: 1811,\tdt: 2.806ms,\tSimpleTD/loss: 0.00343,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00205\n",
      "INFO:TrainMonitor:ep: 980,\tT: 1,950,525,\tG: 17,\tavg_r: 0.00895,\tavg_G: 18.1,\tt: 1899,\tdt: 6.182ms,\tSimpleTD/loss: 0.00447,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00395\n",
      "INFO:TrainMonitor:ep: 981,\tT: 1,952,541,\tG: 15,\tavg_r: 0.00744,\tavg_G: 17.8,\tt: 2015,\tdt: 2.821ms,\tSimpleTD/loss: 0.00675,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00411\n",
      "INFO:TrainMonitor:ep: 982,\tT: 1,954,587,\tG: 15,\tavg_r: 0.00733,\tavg_G: 17.5,\tt: 2045,\tdt: 2.808ms,\tSimpleTD/loss: 0.00743,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00459\n",
      "INFO:TrainMonitor:ep: 983,\tT: 1,956,670,\tG: 15,\tavg_r: 0.0072,\tavg_G: 17.2,\tt: 2082,\tdt: 2.817ms,\tSimpleTD/loss: 0.00744,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00327\n",
      "INFO:TrainMonitor:ep: 984,\tT: 1,958,671,\tG: 15,\tavg_r: 0.0075,\tavg_G: 17,\tt: 2000,\tdt: 2.819ms,\tSimpleTD/loss: 0.00635,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00445\n",
      "INFO:TrainMonitor:ep: 985,\tT: 1,960,472,\tG: 20,\tavg_r: 0.0111,\tavg_G: 17.3,\tt: 1800,\tdt: 6.317ms,\tSimpleTD/loss: 0.00645,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00456\n",
      "INFO:TrainMonitor:ep: 986,\tT: 1,962,375,\tG: 15,\tavg_r: 0.00789,\tavg_G: 17.1,\tt: 1902,\tdt: 2.808ms,\tSimpleTD/loss: 0.00649,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00256\n",
      "INFO:TrainMonitor:ep: 987,\tT: 1,964,075,\tG: 19,\tavg_r: 0.0112,\tavg_G: 17.3,\tt: 1699,\tdt: 2.828ms,\tSimpleTD/loss: 0.00143,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00176\n",
      "INFO:TrainMonitor:ep: 988,\tT: 1,965,733,\tG: 20,\tavg_r: 0.0121,\tavg_G: 17.5,\tt: 1657,\tdt: 2.786ms,\tSimpleTD/loss: 0.00124,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00291\n",
      "INFO:TrainMonitor:ep: 989,\tT: 1,967,394,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.8,\tt: 1660,\tdt: 2.843ms,\tSimpleTD/loss: 0.00135,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00231\n",
      "INFO:TrainMonitor:ep: 990,\tT: 1,969,138,\tG: 20,\tavg_r: 0.0115,\tavg_G: 18,\tt: 1743,\tdt: 2.781ms,\tSimpleTD/loss: 0.00143,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00287\n",
      "INFO:TrainMonitor:ep: 991,\tT: 1,970,798,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.2,\tt: 1659,\tdt: 6.107ms,\tSimpleTD/loss: 0.000792,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00232\n",
      "INFO:TrainMonitor:ep: 992,\tT: 1,972,553,\tG: 18,\tavg_r: 0.0103,\tavg_G: 18.2,\tt: 1754,\tdt: 2.834ms,\tSimpleTD/loss: 0.00213,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00273\n",
      "INFO:TrainMonitor:ep: 993,\tT: 1,974,214,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.4,\tt: 1660,\tdt: 2.791ms,\tSimpleTD/loss: 0.00152,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00237\n",
      "INFO:TrainMonitor:ep: 994,\tT: 1,976,032,\tG: 18,\tavg_r: 0.00991,\tavg_G: 18.3,\tt: 1817,\tdt: 2.823ms,\tSimpleTD/loss: 0.0026,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.000799\n",
      "INFO:TrainMonitor:ep: 995,\tT: 1,977,786,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18.4,\tt: 1753,\tdt: 2.826ms,\tSimpleTD/loss: 0.00256,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00323\n",
      "INFO:TrainMonitor:ep: 996,\tT: 1,979,512,\tG: 19,\tavg_r: 0.011,\tavg_G: 18.5,\tt: 1725,\tdt: 2.827ms,\tSimpleTD/loss: 0.00181,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0021\n",
      "INFO:TrainMonitor:ep: 997,\tT: 1,981,545,\tG: 14,\tavg_r: 0.00689,\tavg_G: 18,\tt: 2032,\tdt: 5.384ms,\tSimpleTD/loss: 0.00766,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00453\n",
      "INFO:TrainMonitor:ep: 998,\tT: 1,983,681,\tG: 13,\tavg_r: 0.00609,\tavg_G: 17.5,\tt: 2135,\tdt: 2.811ms,\tSimpleTD/loss: 0.006,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0026\n",
      "INFO:TrainMonitor:ep: 999,\tT: 1,985,349,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.8,\tt: 1667,\tdt: 2.915ms,\tSimpleTD/loss: 0.00147,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00311\n",
      "INFO:TrainMonitor:ep: 1000,\tT: 1,987,013,\tG: 20,\tavg_r: 0.012,\tavg_G: 18,\tt: 1663,\tdt: 2.784ms,\tSimpleTD/loss: 0.000863,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00264\n",
      "INFO:TrainMonitor:ep: 1001,\tT: 1,988,877,\tG: 16,\tavg_r: 0.00859,\tavg_G: 17.8,\tt: 1863,\tdt: 2.811ms,\tSimpleTD/loss: 0.00546,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00392\n",
      "INFO:TrainMonitor:ep: 1002,\tT: 1,990,653,\tG: 18,\tavg_r: 0.0101,\tavg_G: 17.8,\tt: 1775,\tdt: 5.946ms,\tSimpleTD/loss: 0.00334,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00337\n",
      "INFO:TrainMonitor:ep: 1003,\tT: 1,992,331,\tG: 20,\tavg_r: 0.0119,\tavg_G: 18,\tt: 1677,\tdt: 2.821ms,\tSimpleTD/loss: 0.00126,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00276\n",
      "INFO:TrainMonitor:ep: 1004,\tT: 1,993,992,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.2,\tt: 1660,\tdt: 2.812ms,\tSimpleTD/loss: 0.000621,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.0025\n",
      "INFO:TrainMonitor:ep: 1005,\tT: 1,995,713,\tG: 19,\tavg_r: 0.011,\tavg_G: 18.3,\tt: 1720,\tdt: 2.820ms,\tSimpleTD/loss: 0.00136,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00204\n",
      "INFO:TrainMonitor:ep: 1006,\tT: 1,997,374,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.5,\tt: 1660,\tdt: 2.847ms,\tSimpleTD/loss: 0.000646,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00262\n",
      "INFO:TrainMonitor:ep: 1007,\tT: 1,999,085,\tG: 20,\tavg_r: 0.0117,\tavg_G: 18.6,\tt: 1710,\tdt: 2.775ms,\tSimpleTD/loss: 0.00322,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00331\n",
      "INFO:TrainMonitor:ep: 1008,\tT: 2,000,848,\tG: 18,\tavg_r: 0.0102,\tavg_G: 18.6,\tt: 1762,\tdt: 5.790ms,\tSimpleTD/loss: 0.00378,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00123\n",
      "INFO:TrainMonitor:ep: 1009,\tT: 2,002,754,\tG: 18,\tavg_r: 0.00945,\tavg_G: 18.5,\tt: 1905,\tdt: 2.831ms,\tSimpleTD/loss: 0.00414,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00305\n",
      "INFO:TrainMonitor:ep: 1010,\tT: 2,004,632,\tG: 17,\tavg_r: 0.00906,\tavg_G: 18.4,\tt: 1877,\tdt: 2.790ms,\tSimpleTD/loss: 0.00392,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00217\n",
      "INFO:TrainMonitor:ep: 1011,\tT: 2,006,492,\tG: 18,\tavg_r: 0.00968,\tavg_G: 18.3,\tt: 1859,\tdt: 2.811ms,\tSimpleTD/loss: 0.00178,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00305\n",
      "INFO:TrainMonitor:ep: 1012,\tT: 2,008,153,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.5,\tt: 1660,\tdt: 2.885ms,\tSimpleTD/loss: 0.00172,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00245\n",
      "INFO:TrainMonitor:ep: 1013,\tT: 2,009,910,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18.5,\tt: 1756,\tdt: 2.800ms,\tSimpleTD/loss: 0.00453,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00287\n",
      "INFO:TrainMonitor:ep: 1014,\tT: 2,011,824,\tG: 16,\tavg_r: 0.00836,\tavg_G: 18.3,\tt: 1913,\tdt: 6.081ms,\tSimpleTD/loss: 0.00449,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00204\n",
      "INFO:TrainMonitor:ep: 1015,\tT: 2,013,902,\tG: 15,\tavg_r: 0.00722,\tavg_G: 18,\tt: 2077,\tdt: 2.801ms,\tSimpleTD/loss: 0.00454,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00387\n",
      "INFO:TrainMonitor:ep: 1016,\tT: 2,016,533,\tG: -2,\tavg_r: -0.00076,\tavg_G: 16,\tt: 2630,\tdt: 2.787ms,\tSimpleTD/loss: 0.0132,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00486\n",
      "INFO:TrainMonitor:ep: 1017,\tT: 2,018,316,\tG: 19,\tavg_r: 0.0107,\tavg_G: 16.3,\tt: 1782,\tdt: 2.809ms,\tSimpleTD/loss: 0.00584,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.000656\n",
      "INFO:TrainMonitor:ep: 1018,\tT: 2,020,154,\tG: 17,\tavg_r: 0.00925,\tavg_G: 16.3,\tt: 1837,\tdt: 6.601ms,\tSimpleTD/loss: 0.00427,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00236\n",
      "INFO:TrainMonitor:ep: 1019,\tT: 2,021,926,\tG: 18,\tavg_r: 0.0102,\tavg_G: 16.5,\tt: 1771,\tdt: 2.774ms,\tSimpleTD/loss: 0.00375,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.000956\n",
      "INFO:TrainMonitor:ep: 1020,\tT: 2,023,887,\tG: 16,\tavg_r: 0.00816,\tavg_G: 16.5,\tt: 1960,\tdt: 2.849ms,\tSimpleTD/loss: 0.00784,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00497\n",
      "INFO:TrainMonitor:ep: 1021,\tT: 2,025,549,\tG: 20,\tavg_r: 0.012,\tavg_G: 16.8,\tt: 1661,\tdt: 2.841ms,\tSimpleTD/loss: 0.00168,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.0019\n",
      "INFO:TrainMonitor:ep: 1022,\tT: 2,027,580,\tG: 14,\tavg_r: 0.0069,\tavg_G: 16.5,\tt: 2030,\tdt: 2.824ms,\tSimpleTD/loss: 0.00796,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00485\n",
      "INFO:TrainMonitor:ep: 1023,\tT: 2,029,242,\tG: 20,\tavg_r: 0.012,\tavg_G: 16.9,\tt: 1661,\tdt: 2.787ms,\tSimpleTD/loss: 0.00243,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00538\n",
      "INFO:TrainMonitor:ep: 1024,\tT: 2,030,959,\tG: 19,\tavg_r: 0.0111,\tavg_G: 17.1,\tt: 1716,\tdt: 5.891ms,\tSimpleTD/loss: 0.00287,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0018\n",
      "INFO:TrainMonitor:ep: 1025,\tT: 2,032,727,\tG: 18,\tavg_r: 0.0102,\tavg_G: 17.2,\tt: 1767,\tdt: 2.801ms,\tSimpleTD/loss: 0.004,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00297\n",
      "INFO:TrainMonitor:ep: 1026,\tT: 2,034,588,\tG: 17,\tavg_r: 0.00914,\tavg_G: 17.2,\tt: 1860,\tdt: 2.788ms,\tSimpleTD/loss: 0.00485,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.0038\n",
      "INFO:TrainMonitor:ep: 1027,\tT: 2,036,338,\tG: 19,\tavg_r: 0.0109,\tavg_G: 17.3,\tt: 1749,\tdt: 2.812ms,\tSimpleTD/loss: 0.00274,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00265\n",
      "INFO:TrainMonitor:ep: 1028,\tT: 2,038,101,\tG: 19,\tavg_r: 0.0108,\tavg_G: 17.5,\tt: 1762,\tdt: 2.838ms,\tSimpleTD/loss: 0.00187,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00215\n",
      "INFO:TrainMonitor:ep: 1029,\tT: 2,039,954,\tG: 17,\tavg_r: 0.00918,\tavg_G: 17.5,\tt: 1852,\tdt: 2.808ms,\tSimpleTD/loss: 0.00548,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.000932\n",
      "INFO:TrainMonitor:ep: 1030,\tT: 2,041,716,\tG: 19,\tavg_r: 0.0108,\tavg_G: 17.6,\tt: 1761,\tdt: 5.782ms,\tSimpleTD/loss: 0.00189,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00359\n",
      "INFO:TrainMonitor:ep: 1031,\tT: 2,043,924,\tG: 12,\tavg_r: 0.00544,\tavg_G: 17.1,\tt: 2207,\tdt: 2.784ms,\tSimpleTD/loss: 0.00748,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00348\n",
      "INFO:TrainMonitor:ep: 1032,\tT: 2,045,600,\tG: 20,\tavg_r: 0.0119,\tavg_G: 17.3,\tt: 1675,\tdt: 2.819ms,\tSimpleTD/loss: 0.00155,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00366\n",
      "INFO:TrainMonitor:ep: 1033,\tT: 2,047,528,\tG: 17,\tavg_r: 0.00882,\tavg_G: 17.3,\tt: 1927,\tdt: 2.779ms,\tSimpleTD/loss: 0.00506,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00229\n",
      "INFO:TrainMonitor:ep: 1034,\tT: 2,049,196,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.6,\tt: 1667,\tdt: 2.824ms,\tSimpleTD/loss: 0.00177,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00389\n",
      "INFO:TrainMonitor:ep: 1035,\tT: 2,050,859,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.8,\tt: 1662,\tdt: 5.935ms,\tSimpleTD/loss: 0.000686,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00272\n",
      "INFO:TrainMonitor:ep: 1036,\tT: 2,052,540,\tG: 20,\tavg_r: 0.0119,\tavg_G: 18,\tt: 1680,\tdt: 2.820ms,\tSimpleTD/loss: 0.00109,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.0027\n",
      "INFO:TrainMonitor:ep: 1037,\tT: 2,054,207,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.2,\tt: 1666,\tdt: 2.820ms,\tSimpleTD/loss: 0.00053,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00222\n",
      "INFO:TrainMonitor:ep: 1038,\tT: 2,055,891,\tG: 20,\tavg_r: 0.0119,\tavg_G: 18.4,\tt: 1683,\tdt: 2.771ms,\tSimpleTD/loss: 0.00125,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00248\n",
      "INFO:TrainMonitor:ep: 1039,\tT: 2,057,600,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.5,\tt: 1708,\tdt: 2.827ms,\tSimpleTD/loss: 0.00354,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00229\n",
      "INFO:TrainMonitor:ep: 1040,\tT: 2,059,257,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.6,\tt: 1656,\tdt: 2.775ms,\tSimpleTD/loss: 0.000758,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00279\n",
      "INFO:TrainMonitor:ep: 1041,\tT: 2,061,011,\tG: 18,\tavg_r: 0.0103,\tavg_G: 18.6,\tt: 1753,\tdt: 5.830ms,\tSimpleTD/loss: 0.0029,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00168\n",
      "INFO:TrainMonitor:ep: 1042,\tT: 2,062,838,\tG: 16,\tavg_r: 0.00876,\tavg_G: 18.3,\tt: 1826,\tdt: 2.802ms,\tSimpleTD/loss: 0.00578,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00289\n",
      "INFO:TrainMonitor:ep: 1043,\tT: 2,064,501,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.5,\tt: 1662,\tdt: 2.829ms,\tSimpleTD/loss: 0.00121,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00201\n",
      "INFO:TrainMonitor:ep: 1044,\tT: 2,066,163,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.6,\tt: 1661,\tdt: 2.773ms,\tSimpleTD/loss: 0.000694,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00286\n",
      "INFO:TrainMonitor:ep: 1045,\tT: 2,068,011,\tG: 17,\tavg_r: 0.0092,\tavg_G: 18.5,\tt: 1847,\tdt: 2.798ms,\tSimpleTD/loss: 0.00706,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.0031\n",
      "INFO:TrainMonitor:ep: 1046,\tT: 2,069,807,\tG: 19,\tavg_r: 0.0106,\tavg_G: 18.5,\tt: 1795,\tdt: 2.813ms,\tSimpleTD/loss: 0.00237,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00238\n",
      "INFO:TrainMonitor:ep: 1047,\tT: 2,071,466,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.7,\tt: 1658,\tdt: 6.167ms,\tSimpleTD/loss: 0.000608,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00298\n",
      "INFO:TrainMonitor:ep: 1048,\tT: 2,073,125,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.8,\tt: 1658,\tdt: 2.777ms,\tSimpleTD/loss: 0.000401,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00263\n",
      "INFO:TrainMonitor:ep: 1049,\tT: 2,074,943,\tG: 17,\tavg_r: 0.00936,\tavg_G: 18.6,\tt: 1817,\tdt: 2.825ms,\tSimpleTD/loss: 0.0052,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00345\n",
      "INFO:TrainMonitor:ep: 1050,\tT: 2,076,605,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.8,\tt: 1661,\tdt: 2.862ms,\tSimpleTD/loss: 0.00117,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00238\n",
      "INFO:TrainMonitor:ep: 1051,\tT: 2,078,332,\tG: 19,\tavg_r: 0.011,\tavg_G: 18.8,\tt: 1726,\tdt: 2.801ms,\tSimpleTD/loss: 0.00158,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00262\n",
      "INFO:TrainMonitor:ep: 1052,\tT: 2,079,995,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.9,\tt: 1662,\tdt: 2.801ms,\tSimpleTD/loss: 0.000433,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00208\n",
      "INFO:TrainMonitor:ep: 1053,\tT: 2,081,958,\tG: 14,\tavg_r: 0.00714,\tavg_G: 18.4,\tt: 1962,\tdt: 5.615ms,\tSimpleTD/loss: 0.00451,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00302\n",
      "INFO:TrainMonitor:ep: 1054,\tT: 2,083,722,\tG: 18,\tavg_r: 0.0102,\tavg_G: 18.4,\tt: 1763,\tdt: 2.809ms,\tSimpleTD/loss: 0.00379,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00271\n",
      "INFO:TrainMonitor:ep: 1055,\tT: 2,085,681,\tG: 17,\tavg_r: 0.00868,\tavg_G: 18.2,\tt: 1958,\tdt: 2.776ms,\tSimpleTD/loss: 0.00414,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00393\n",
      "INFO:TrainMonitor:ep: 1056,\tT: 2,087,504,\tG: 17,\tavg_r: 0.00933,\tavg_G: 18.1,\tt: 1822,\tdt: 2.796ms,\tSimpleTD/loss: 0.00264,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00285\n",
      "INFO:TrainMonitor:ep: 1057,\tT: 2,089,225,\tG: 19,\tavg_r: 0.011,\tavg_G: 18.2,\tt: 1720,\tdt: 2.818ms,\tSimpleTD/loss: 0.00317,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00365\n",
      "INFO:TrainMonitor:ep: 1058,\tT: 2,091,054,\tG: 19,\tavg_r: 0.0104,\tavg_G: 18.3,\tt: 1828,\tdt: 6.074ms,\tSimpleTD/loss: 0.00253,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00315\n",
      "INFO:TrainMonitor:ep: 1059,\tT: 2,092,715,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.5,\tt: 1660,\tdt: 2.826ms,\tSimpleTD/loss: 0.000959,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00238\n",
      "INFO:TrainMonitor:ep: 1060,\tT: 2,094,479,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18.5,\tt: 1763,\tdt: 2.810ms,\tSimpleTD/loss: 0.0019,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00264\n",
      "INFO:TrainMonitor:ep: 1061,\tT: 2,096,142,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.7,\tt: 1662,\tdt: 2.780ms,\tSimpleTD/loss: 0.000613,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00222\n",
      "INFO:TrainMonitor:ep: 1062,\tT: 2,097,887,\tG: 19,\tavg_r: 0.0109,\tavg_G: 18.7,\tt: 1744,\tdt: 2.811ms,\tSimpleTD/loss: 0.00235,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00294\n",
      "INFO:TrainMonitor:ep: 1063,\tT: 2,099,548,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.8,\tt: 1660,\tdt: 2.822ms,\tSimpleTD/loss: 0.000593,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00247\n",
      "INFO:TrainMonitor:ep: 1064,\tT: 2,101,254,\tG: 20,\tavg_r: 0.0117,\tavg_G: 18.9,\tt: 1705,\tdt: 6.012ms,\tSimpleTD/loss: 0.00186,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00254\n",
      "INFO:TrainMonitor:ep: 1065,\tT: 2,103,120,\tG: 17,\tavg_r: 0.00912,\tavg_G: 18.7,\tt: 1865,\tdt: 2.820ms,\tSimpleTD/loss: 0.00473,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00165\n",
      "INFO:TrainMonitor:ep: 1066,\tT: 2,104,935,\tG: 19,\tavg_r: 0.0105,\tavg_G: 18.8,\tt: 1814,\tdt: 2.846ms,\tSimpleTD/loss: 0.00308,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00281\n",
      "INFO:TrainMonitor:ep: 1067,\tT: 2,106,727,\tG: 19,\tavg_r: 0.0106,\tavg_G: 18.8,\tt: 1791,\tdt: 2.799ms,\tSimpleTD/loss: 0.00171,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00266\n",
      "INFO:TrainMonitor:ep: 1068,\tT: 2,108,492,\tG: 18,\tavg_r: 0.0102,\tavg_G: 18.7,\tt: 1764,\tdt: 2.765ms,\tSimpleTD/loss: 0.00465,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00299\n",
      "INFO:TrainMonitor:ep: 1069,\tT: 2,110,154,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.8,\tt: 1661,\tdt: 5.955ms,\tSimpleTD/loss: 0.000596,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00243\n",
      "INFO:TrainMonitor:ep: 1070,\tT: 2,111,937,\tG: 20,\tavg_r: 0.0112,\tavg_G: 19,\tt: 1782,\tdt: 2.790ms,\tSimpleTD/loss: 0.00103,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00237\n",
      "INFO:TrainMonitor:ep: 1071,\tT: 2,113,599,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.1,\tt: 1661,\tdt: 2.773ms,\tSimpleTD/loss: 0.000621,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00229\n",
      "INFO:TrainMonitor:ep: 1072,\tT: 2,115,344,\tG: 20,\tavg_r: 0.0115,\tavg_G: 19.2,\tt: 1744,\tdt: 2.792ms,\tSimpleTD/loss: 0.000801,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00247\n",
      "INFO:TrainMonitor:ep: 1073,\tT: 2,117,021,\tG: 20,\tavg_r: 0.0119,\tavg_G: 19.2,\tt: 1676,\tdt: 2.814ms,\tSimpleTD/loss: 0.00131,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00231\n",
      "INFO:TrainMonitor:ep: 1074,\tT: 2,118,705,\tG: 20,\tavg_r: 0.0119,\tavg_G: 19.3,\tt: 1683,\tdt: 2.793ms,\tSimpleTD/loss: 0.000847,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00272\n",
      "INFO:TrainMonitor:ep: 1075,\tT: 2,120,363,\tG: 20,\tavg_r: 0.0121,\tavg_G: 19.4,\tt: 1657,\tdt: 6.019ms,\tSimpleTD/loss: 0.000455,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00198\n",
      "INFO:TrainMonitor:ep: 1076,\tT: 2,122,027,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.4,\tt: 1663,\tdt: 2.782ms,\tSimpleTD/loss: 0.000314,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00195\n",
      "INFO:TrainMonitor:ep: 1077,\tT: 2,123,691,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.5,\tt: 1663,\tdt: 2.861ms,\tSimpleTD/loss: 0.000329,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00191\n",
      "INFO:TrainMonitor:ep: 1078,\tT: 2,125,417,\tG: 19,\tavg_r: 0.011,\tavg_G: 19.5,\tt: 1725,\tdt: 2.767ms,\tSimpleTD/loss: 0.00128,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00248\n",
      "INFO:TrainMonitor:ep: 1079,\tT: 2,127,200,\tG: 18,\tavg_r: 0.0101,\tavg_G: 19.3,\tt: 1782,\tdt: 2.813ms,\tSimpleTD/loss: 0.002,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00183\n",
      "INFO:TrainMonitor:ep: 1080,\tT: 2,128,866,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.4,\tt: 1665,\tdt: 2.843ms,\tSimpleTD/loss: 0.000855,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00247\n",
      "INFO:TrainMonitor:ep: 1081,\tT: 2,130,611,\tG: 19,\tavg_r: 0.0109,\tavg_G: 19.3,\tt: 1744,\tdt: 6.058ms,\tSimpleTD/loss: 0.00104,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00255\n",
      "INFO:TrainMonitor:ep: 1082,\tT: 2,132,451,\tG: 16,\tavg_r: 0.0087,\tavg_G: 19,\tt: 1839,\tdt: 2.802ms,\tSimpleTD/loss: 0.00533,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00232\n",
      "INFO:TrainMonitor:ep: 1083,\tT: 2,134,246,\tG: 19,\tavg_r: 0.0106,\tavg_G: 19,\tt: 1794,\tdt: 2.805ms,\tSimpleTD/loss: 0.00179,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00277\n",
      "INFO:TrainMonitor:ep: 1084,\tT: 2,136,064,\tG: 18,\tavg_r: 0.00991,\tavg_G: 18.9,\tt: 1817,\tdt: 2.796ms,\tSimpleTD/loss: 0.00468,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.0036\n",
      "INFO:TrainMonitor:ep: 1085,\tT: 2,137,781,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.9,\tt: 1716,\tdt: 2.814ms,\tSimpleTD/loss: 0.00132,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00281\n",
      "INFO:TrainMonitor:ep: 1086,\tT: 2,139,443,\tG: 20,\tavg_r: 0.012,\tavg_G: 19,\tt: 1661,\tdt: 2.781ms,\tSimpleTD/loss: 0.000967,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00237\n",
      "INFO:TrainMonitor:ep: 1087,\tT: 2,141,164,\tG: 19,\tavg_r: 0.011,\tavg_G: 19,\tt: 1720,\tdt: 6.073ms,\tSimpleTD/loss: 0.00258,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00227\n",
      "INFO:TrainMonitor:ep: 1088,\tT: 2,142,916,\tG: 19,\tavg_r: 0.0109,\tavg_G: 19,\tt: 1751,\tdt: 2.794ms,\tSimpleTD/loss: 0.00187,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00197\n",
      "INFO:TrainMonitor:ep: 1089,\tT: 2,144,575,\tG: 20,\tavg_r: 0.0121,\tavg_G: 19.1,\tt: 1658,\tdt: 2.778ms,\tSimpleTD/loss: 0.000726,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00231\n",
      "INFO:TrainMonitor:ep: 1090,\tT: 2,146,295,\tG: 19,\tavg_r: 0.0111,\tavg_G: 19.1,\tt: 1719,\tdt: 2.822ms,\tSimpleTD/loss: 0.00134,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00139\n",
      "INFO:TrainMonitor:ep: 1091,\tT: 2,147,962,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.2,\tt: 1666,\tdt: 2.828ms,\tSimpleTD/loss: 0.00144,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00242\n",
      "INFO:TrainMonitor:ep: 1092,\tT: 2,149,641,\tG: 20,\tavg_r: 0.0119,\tavg_G: 19.3,\tt: 1678,\tdt: 2.765ms,\tSimpleTD/loss: 0.00099,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00289\n",
      "INFO:TrainMonitor:ep: 1093,\tT: 2,151,329,\tG: 21,\tavg_r: 0.0124,\tavg_G: 19.4,\tt: 1687,\tdt: 5.903ms,\tSimpleTD/loss: 0.00158,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.002\n",
      "INFO:TrainMonitor:ep: 1094,\tT: 2,152,995,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.5,\tt: 1665,\tdt: 2.772ms,\tSimpleTD/loss: 0.000711,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00214\n",
      "INFO:TrainMonitor:ep: 1095,\tT: 2,154,726,\tG: 19,\tavg_r: 0.011,\tavg_G: 19.5,\tt: 1730,\tdt: 2.823ms,\tSimpleTD/loss: 0.00232,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00188\n",
      "INFO:TrainMonitor:ep: 1096,\tT: 2,156,574,\tG: 18,\tavg_r: 0.00975,\tavg_G: 19.3,\tt: 1847,\tdt: 2.790ms,\tSimpleTD/loss: 0.00272,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.0031\n",
      "INFO:TrainMonitor:ep: 1097,\tT: 2,158,234,\tG: 20,\tavg_r: 0.0121,\tavg_G: 19.4,\tt: 1659,\tdt: 2.834ms,\tSimpleTD/loss: 0.00163,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00232\n",
      "INFO:TrainMonitor:ep: 1098,\tT: 2,159,936,\tG: 20,\tavg_r: 0.0118,\tavg_G: 19.4,\tt: 1701,\tdt: 2.785ms,\tSimpleTD/loss: 0.000769,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.0022\n",
      "INFO:TrainMonitor:ep: 1099,\tT: 2,161,597,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.5,\tt: 1660,\tdt: 6.007ms,\tSimpleTD/loss: 0.000418,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00216\n",
      "INFO:TrainMonitor:ep: 1100,\tT: 2,163,341,\tG: 20,\tavg_r: 0.0115,\tavg_G: 19.5,\tt: 1743,\tdt: 2.803ms,\tSimpleTD/loss: 0.000889,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00231\n",
      "INFO:TrainMonitor:ep: 1101,\tT: 2,165,026,\tG: 20,\tavg_r: 0.0119,\tavg_G: 19.6,\tt: 1684,\tdt: 2.762ms,\tSimpleTD/loss: 0.000491,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00236\n",
      "INFO:TrainMonitor:ep: 1102,\tT: 2,166,814,\tG: 18,\tavg_r: 0.0101,\tavg_G: 19.4,\tt: 1787,\tdt: 2.807ms,\tSimpleTD/loss: 0.00244,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00255\n",
      "INFO:TrainMonitor:ep: 1103,\tT: 2,168,618,\tG: 17,\tavg_r: 0.00943,\tavg_G: 19.2,\tt: 1803,\tdt: 2.787ms,\tSimpleTD/loss: 0.00184,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00209\n",
      "INFO:TrainMonitor:ep: 1104,\tT: 2,170,281,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.3,\tt: 1662,\tdt: 6.031ms,\tSimpleTD/loss: 0.000605,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00247\n",
      "INFO:TrainMonitor:ep: 1105,\tT: 2,171,968,\tG: 21,\tavg_r: 0.0125,\tavg_G: 19.4,\tt: 1686,\tdt: 2.771ms,\tSimpleTD/loss: 0.00077,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00195\n",
      "INFO:TrainMonitor:ep: 1106,\tT: 2,173,626,\tG: 20,\tavg_r: 0.0121,\tavg_G: 19.5,\tt: 1657,\tdt: 2.818ms,\tSimpleTD/loss: 0.000623,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00254\n",
      "INFO:TrainMonitor:ep: 1107,\tT: 2,175,381,\tG: 19,\tavg_r: 0.0108,\tavg_G: 19.4,\tt: 1754,\tdt: 2.794ms,\tSimpleTD/loss: 0.00363,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00132\n",
      "INFO:TrainMonitor:ep: 1108,\tT: 2,177,095,\tG: 19,\tavg_r: 0.0111,\tavg_G: 19.4,\tt: 1713,\tdt: 2.772ms,\tSimpleTD/loss: 0.00129,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00218\n",
      "INFO:TrainMonitor:ep: 1109,\tT: 2,178,756,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.5,\tt: 1660,\tdt: 2.826ms,\tSimpleTD/loss: 0.000396,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00221\n",
      "INFO:TrainMonitor:ep: 1110,\tT: 2,180,588,\tG: 17,\tavg_r: 0.00928,\tavg_G: 19.2,\tt: 1831,\tdt: 5.659ms,\tSimpleTD/loss: 0.00407,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0014\n",
      "INFO:TrainMonitor:ep: 1111,\tT: 2,182,219,\tG: 21,\tavg_r: 0.0129,\tavg_G: 19.4,\tt: 1630,\tdt: 2.789ms,\tSimpleTD/loss: 0.00156,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00258\n",
      "INFO:TrainMonitor:ep: 1112,\tT: 2,183,932,\tG: 19,\tavg_r: 0.0111,\tavg_G: 19.4,\tt: 1712,\tdt: 2.807ms,\tSimpleTD/loss: 0.00151,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00271\n",
      "INFO:TrainMonitor:ep: 1113,\tT: 2,185,628,\tG: 19,\tavg_r: 0.0112,\tavg_G: 19.3,\tt: 1695,\tdt: 2.829ms,\tSimpleTD/loss: 0.000672,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00229\n",
      "INFO:TrainMonitor:ep: 1114,\tT: 2,187,257,\tG: 21,\tavg_r: 0.0129,\tavg_G: 19.5,\tt: 1628,\tdt: 2.782ms,\tSimpleTD/loss: 0.000851,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.0024\n",
      "INFO:TrainMonitor:ep: 1115,\tT: 2,189,044,\tG: 18,\tavg_r: 0.0101,\tavg_G: 19.3,\tt: 1786,\tdt: 2.795ms,\tSimpleTD/loss: 0.00221,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.0026\n",
      "INFO:TrainMonitor:ep: 1116,\tT: 2,190,799,\tG: 19,\tavg_r: 0.0108,\tavg_G: 19.3,\tt: 1754,\tdt: 6.176ms,\tSimpleTD/loss: 0.00153,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00258\n",
      "INFO:TrainMonitor:ep: 1117,\tT: 2,192,524,\tG: 19,\tavg_r: 0.011,\tavg_G: 19.3,\tt: 1724,\tdt: 2.771ms,\tSimpleTD/loss: 0.0014,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00263\n",
      "INFO:TrainMonitor:ep: 1118,\tT: 2,194,327,\tG: 18,\tavg_r: 0.00999,\tavg_G: 19.1,\tt: 1802,\tdt: 2.846ms,\tSimpleTD/loss: 0.00171,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00151\n",
      "INFO:TrainMonitor:ep: 1119,\tT: 2,195,990,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.2,\tt: 1662,\tdt: 2.799ms,\tSimpleTD/loss: 0.000621,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00244\n",
      "INFO:TrainMonitor:ep: 1120,\tT: 2,197,749,\tG: 19,\tavg_r: 0.0108,\tavg_G: 19.2,\tt: 1758,\tdt: 2.814ms,\tSimpleTD/loss: 0.00108,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00229\n",
      "INFO:TrainMonitor:ep: 1121,\tT: 2,199,630,\tG: 17,\tavg_r: 0.00904,\tavg_G: 19,\tt: 1880,\tdt: 2.797ms,\tSimpleTD/loss: 0.00423,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00304\n",
      "INFO:TrainMonitor:ep: 1122,\tT: 2,201,354,\tG: 19,\tavg_r: 0.011,\tavg_G: 19,\tt: 1723,\tdt: 6.328ms,\tSimpleTD/loss: 0.00142,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00211\n",
      "INFO:TrainMonitor:ep: 1123,\tT: 2,203,038,\tG: 20,\tavg_r: 0.0119,\tavg_G: 19.1,\tt: 1683,\tdt: 2.828ms,\tSimpleTD/loss: 0.00147,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00273\n",
      "INFO:TrainMonitor:ep: 1124,\tT: 2,204,762,\tG: 19,\tavg_r: 0.011,\tavg_G: 19.1,\tt: 1723,\tdt: 2.773ms,\tSimpleTD/loss: 0.00186,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00284\n",
      "INFO:TrainMonitor:ep: 1125,\tT: 2,206,422,\tG: 20,\tavg_r: 0.0121,\tavg_G: 19.2,\tt: 1659,\tdt: 2.827ms,\tSimpleTD/loss: 0.000634,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.002\n",
      "INFO:TrainMonitor:ep: 1126,\tT: 2,208,154,\tG: 18,\tavg_r: 0.0104,\tavg_G: 19.1,\tt: 1731,\tdt: 2.778ms,\tSimpleTD/loss: 0.0027,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00249\n",
      "INFO:TrainMonitor:ep: 1127,\tT: 2,210,128,\tG: 15,\tavg_r: 0.0076,\tavg_G: 18.7,\tt: 1973,\tdt: 5.837ms,\tSimpleTD/loss: 0.00506,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00476\n",
      "INFO:TrainMonitor:ep: 1128,\tT: 2,211,997,\tG: 16,\tavg_r: 0.00857,\tavg_G: 18.4,\tt: 1868,\tdt: 2.784ms,\tSimpleTD/loss: 0.00316,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00153\n",
      "INFO:TrainMonitor:ep: 1129,\tT: 2,213,763,\tG: 18,\tavg_r: 0.0102,\tavg_G: 18.3,\tt: 1765,\tdt: 2.801ms,\tSimpleTD/loss: 0.00389,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00303\n",
      "INFO:TrainMonitor:ep: 1130,\tT: 2,215,462,\tG: 19,\tavg_r: 0.0112,\tavg_G: 18.4,\tt: 1698,\tdt: 2.805ms,\tSimpleTD/loss: 0.00283,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0039\n",
      "INFO:TrainMonitor:ep: 1131,\tT: 2,217,120,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.6,\tt: 1657,\tdt: 2.764ms,\tSimpleTD/loss: 0.000803,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00209\n",
      "INFO:TrainMonitor:ep: 1132,\tT: 2,218,778,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.7,\tt: 1657,\tdt: 2.825ms,\tSimpleTD/loss: 0.000391,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00212\n",
      "INFO:TrainMonitor:ep: 1133,\tT: 2,220,626,\tG: 16,\tavg_r: 0.00866,\tavg_G: 18.4,\tt: 1847,\tdt: 5.622ms,\tSimpleTD/loss: 0.0031,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00283\n",
      "INFO:TrainMonitor:ep: 1134,\tT: 2,222,289,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.6,\tt: 1662,\tdt: 2.844ms,\tSimpleTD/loss: 0.000739,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00202\n",
      "INFO:TrainMonitor:ep: 1135,\tT: 2,223,956,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.7,\tt: 1666,\tdt: 2.760ms,\tSimpleTD/loss: 0.000376,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00216\n",
      "INFO:TrainMonitor:ep: 1136,\tT: 2,225,621,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.9,\tt: 1664,\tdt: 2.813ms,\tSimpleTD/loss: 0.000401,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00216\n",
      "INFO:TrainMonitor:ep: 1137,\tT: 2,227,644,\tG: 14,\tavg_r: 0.00692,\tavg_G: 18.4,\tt: 2022,\tdt: 2.808ms,\tSimpleTD/loss: 0.00548,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.000758\n",
      "INFO:TrainMonitor:ep: 1138,\tT: 2,229,307,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.5,\tt: 1662,\tdt: 2.760ms,\tSimpleTD/loss: 0.000707,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00295\n",
      "INFO:TrainMonitor:ep: 1139,\tT: 2,231,152,\tG: 18,\tavg_r: 0.00976,\tavg_G: 18.5,\tt: 1844,\tdt: 5.612ms,\tSimpleTD/loss: 0.00254,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00209\n",
      "INFO:TrainMonitor:ep: 1140,\tT: 2,232,820,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.6,\tt: 1667,\tdt: 2.819ms,\tSimpleTD/loss: 0.00063,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00223\n",
      "INFO:TrainMonitor:ep: 1141,\tT: 2,234,742,\tG: 17,\tavg_r: 0.00885,\tavg_G: 18.5,\tt: 1921,\tdt: 2.786ms,\tSimpleTD/loss: 0.00642,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00409\n",
      "INFO:TrainMonitor:ep: 1142,\tT: 2,236,441,\tG: 20,\tavg_r: 0.0118,\tavg_G: 18.6,\tt: 1698,\tdt: 2.812ms,\tSimpleTD/loss: 0.00217,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00222\n",
      "INFO:TrainMonitor:ep: 1143,\tT: 2,238,128,\tG: 20,\tavg_r: 0.0119,\tavg_G: 18.8,\tt: 1686,\tdt: 2.768ms,\tSimpleTD/loss: 0.000749,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00237\n",
      "INFO:TrainMonitor:ep: 1144,\tT: 2,240,130,\tG: 16,\tavg_r: 0.008,\tavg_G: 18.5,\tt: 2001,\tdt: 5.898ms,\tSimpleTD/loss: 0.0043,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00208\n",
      "INFO:TrainMonitor:ep: 1145,\tT: 2,242,019,\tG: 15,\tavg_r: 0.00794,\tavg_G: 18.1,\tt: 1888,\tdt: 2.839ms,\tSimpleTD/loss: 0.0061,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.003\n",
      "INFO:TrainMonitor:ep: 1146,\tT: 2,243,721,\tG: 19,\tavg_r: 0.0112,\tavg_G: 18.2,\tt: 1701,\tdt: 2.767ms,\tSimpleTD/loss: 0.00177,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00301\n",
      "INFO:TrainMonitor:ep: 1147,\tT: 2,245,561,\tG: 17,\tavg_r: 0.00924,\tavg_G: 18.1,\tt: 1839,\tdt: 2.834ms,\tSimpleTD/loss: 0.00389,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00326\n",
      "INFO:TrainMonitor:ep: 1148,\tT: 2,247,281,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.2,\tt: 1719,\tdt: 2.766ms,\tSimpleTD/loss: 0.00252,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00303\n",
      "INFO:TrainMonitor:ep: 1149,\tT: 2,249,062,\tG: 19,\tavg_r: 0.0107,\tavg_G: 18.3,\tt: 1780,\tdt: 2.794ms,\tSimpleTD/loss: 0.00448,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00335\n",
      "INFO:TrainMonitor:ep: 1150,\tT: 2,250,719,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.4,\tt: 1656,\tdt: 5.987ms,\tSimpleTD/loss: 0.00279,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00266\n",
      "INFO:TrainMonitor:ep: 1151,\tT: 2,252,413,\tG: 20,\tavg_r: 0.0118,\tavg_G: 18.6,\tt: 1693,\tdt: 2.773ms,\tSimpleTD/loss: 0.000837,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.0019\n",
      "INFO:TrainMonitor:ep: 1152,\tT: 2,254,129,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.6,\tt: 1715,\tdt: 2.830ms,\tSimpleTD/loss: 0.00197,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.0024\n",
      "INFO:TrainMonitor:ep: 1153,\tT: 2,255,890,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18.7,\tt: 1760,\tdt: 2.820ms,\tSimpleTD/loss: 0.00207,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00158\n",
      "INFO:TrainMonitor:ep: 1154,\tT: 2,257,775,\tG: 16,\tavg_r: 0.00849,\tavg_G: 18.4,\tt: 1884,\tdt: 2.816ms,\tSimpleTD/loss: 0.00429,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00337\n",
      "INFO:TrainMonitor:ep: 1155,\tT: 2,259,840,\tG: 13,\tavg_r: 0.0063,\tavg_G: 17.9,\tt: 2064,\tdt: 2.817ms,\tSimpleTD/loss: 0.00639,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00333\n",
      "INFO:TrainMonitor:ep: 1156,\tT: 2,261,785,\tG: 15,\tavg_r: 0.00772,\tavg_G: 17.6,\tt: 1944,\tdt: 5.872ms,\tSimpleTD/loss: 0.00567,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00362\n",
      "INFO:TrainMonitor:ep: 1157,\tT: 2,263,934,\tG: 12,\tavg_r: 0.00559,\tavg_G: 17,\tt: 2148,\tdt: 2.791ms,\tSimpleTD/loss: 0.00658,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00334\n",
      "INFO:TrainMonitor:ep: 1158,\tT: 2,265,693,\tG: 19,\tavg_r: 0.0108,\tavg_G: 17.2,\tt: 1758,\tdt: 2.805ms,\tSimpleTD/loss: 0.00467,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.0027\n",
      "INFO:TrainMonitor:ep: 1159,\tT: 2,267,359,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.5,\tt: 1665,\tdt: 2.814ms,\tSimpleTD/loss: 0.00149,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00279\n",
      "INFO:TrainMonitor:ep: 1160,\tT: 2,269,061,\tG: 19,\tavg_r: 0.0112,\tavg_G: 17.6,\tt: 1701,\tdt: 2.784ms,\tSimpleTD/loss: 0.00247,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00198\n",
      "INFO:TrainMonitor:ep: 1161,\tT: 2,270,764,\tG: 19,\tavg_r: 0.0112,\tavg_G: 17.8,\tt: 1702,\tdt: 6.079ms,\tSimpleTD/loss: 0.00067,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00157\n",
      "INFO:TrainMonitor:ep: 1162,\tT: 2,272,425,\tG: 20,\tavg_r: 0.012,\tavg_G: 18,\tt: 1660,\tdt: 2.771ms,\tSimpleTD/loss: 0.000779,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00261\n",
      "INFO:TrainMonitor:ep: 1163,\tT: 2,274,127,\tG: 20,\tavg_r: 0.0118,\tavg_G: 18.2,\tt: 1701,\tdt: 2.810ms,\tSimpleTD/loss: 0.000801,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00224\n",
      "INFO:TrainMonitor:ep: 1164,\tT: 2,275,835,\tG: 20,\tavg_r: 0.0117,\tavg_G: 18.4,\tt: 1707,\tdt: 2.861ms,\tSimpleTD/loss: 0.00134,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00214\n",
      "INFO:TrainMonitor:ep: 1165,\tT: 2,277,498,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.5,\tt: 1662,\tdt: 2.781ms,\tSimpleTD/loss: 0.000531,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00212\n",
      "INFO:TrainMonitor:ep: 1166,\tT: 2,279,240,\tG: 20,\tavg_r: 0.0115,\tavg_G: 18.7,\tt: 1741,\tdt: 2.807ms,\tSimpleTD/loss: 0.001,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0026\n",
      "INFO:TrainMonitor:ep: 1167,\tT: 2,281,222,\tG: 14,\tavg_r: 0.00707,\tavg_G: 18.2,\tt: 1981,\tdt: 5.648ms,\tSimpleTD/loss: 0.00769,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00218\n",
      "INFO:TrainMonitor:ep: 1168,\tT: 2,283,187,\tG: 16,\tavg_r: 0.00815,\tavg_G: 18,\tt: 1964,\tdt: 2.770ms,\tSimpleTD/loss: 0.0056,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.0037\n",
      "INFO:TrainMonitor:ep: 1169,\tT: 2,285,362,\tG: 11,\tavg_r: 0.00506,\tavg_G: 17.3,\tt: 2174,\tdt: 2.816ms,\tSimpleTD/loss: 0.00836,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00384\n",
      "INFO:TrainMonitor:ep: 1170,\tT: 2,287,025,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.6,\tt: 1662,\tdt: 2.770ms,\tSimpleTD/loss: 0.00157,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00356\n",
      "INFO:TrainMonitor:ep: 1171,\tT: 2,288,822,\tG: 17,\tavg_r: 0.00947,\tavg_G: 17.5,\tt: 1796,\tdt: 2.798ms,\tSimpleTD/loss: 0.00488,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00273\n",
      "INFO:TrainMonitor:ep: 1172,\tT: 2,290,557,\tG: 18,\tavg_r: 0.0104,\tavg_G: 17.6,\tt: 1734,\tdt: 6.334ms,\tSimpleTD/loss: 0.00146,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00217\n",
      "INFO:TrainMonitor:ep: 1173,\tT: 2,292,283,\tG: 20,\tavg_r: 0.0116,\tavg_G: 17.8,\tt: 1725,\tdt: 2.827ms,\tSimpleTD/loss: 0.00145,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00348\n",
      "INFO:TrainMonitor:ep: 1174,\tT: 2,294,040,\tG: 20,\tavg_r: 0.0114,\tavg_G: 18,\tt: 1756,\tdt: 2.802ms,\tSimpleTD/loss: 0.00366,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00269\n",
      "INFO:TrainMonitor:ep: 1175,\tT: 2,295,822,\tG: 19,\tavg_r: 0.0107,\tavg_G: 18.1,\tt: 1781,\tdt: 2.812ms,\tSimpleTD/loss: 0.0033,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.0033\n",
      "INFO:TrainMonitor:ep: 1176,\tT: 2,297,540,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.2,\tt: 1717,\tdt: 2.787ms,\tSimpleTD/loss: 0.000958,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00197\n",
      "INFO:TrainMonitor:ep: 1177,\tT: 2,299,336,\tG: 18,\tavg_r: 0.01,\tavg_G: 18.2,\tt: 1795,\tdt: 2.797ms,\tSimpleTD/loss: 0.00306,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00198\n",
      "INFO:TrainMonitor:ep: 1178,\tT: 2,301,026,\tG: 21,\tavg_r: 0.0124,\tavg_G: 18.5,\tt: 1689,\tdt: 6.266ms,\tSimpleTD/loss: 0.0021,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00233\n",
      "INFO:TrainMonitor:ep: 1179,\tT: 2,303,025,\tG: 15,\tavg_r: 0.00751,\tavg_G: 18.1,\tt: 1998,\tdt: 2.824ms,\tSimpleTD/loss: 0.00466,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.0025\n",
      "INFO:TrainMonitor:ep: 1180,\tT: 2,304,811,\tG: 18,\tavg_r: 0.0101,\tavg_G: 18.1,\tt: 1785,\tdt: 2.833ms,\tSimpleTD/loss: 0.00477,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00375\n",
      "INFO:TrainMonitor:ep: 1181,\tT: 2,306,560,\tG: 18,\tavg_r: 0.0103,\tavg_G: 18.1,\tt: 1748,\tdt: 2.830ms,\tSimpleTD/loss: 0.00257,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00197\n",
      "INFO:TrainMonitor:ep: 1182,\tT: 2,308,232,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.3,\tt: 1671,\tdt: 2.776ms,\tSimpleTD/loss: 0.00291,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00348\n",
      "INFO:TrainMonitor:ep: 1183,\tT: 2,309,958,\tG: 19,\tavg_r: 0.011,\tavg_G: 18.4,\tt: 1725,\tdt: 2.813ms,\tSimpleTD/loss: 0.00224,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00193\n",
      "INFO:TrainMonitor:ep: 1184,\tT: 2,311,656,\tG: 19,\tavg_r: 0.0112,\tavg_G: 18.4,\tt: 1697,\tdt: 5.928ms,\tSimpleTD/loss: 0.000898,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00206\n",
      "INFO:TrainMonitor:ep: 1185,\tT: 2,313,555,\tG: 18,\tavg_r: 0.00948,\tavg_G: 18.4,\tt: 1898,\tdt: 2.787ms,\tSimpleTD/loss: 0.00399,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00262\n",
      "INFO:TrainMonitor:ep: 1186,\tT: 2,315,573,\tG: 15,\tavg_r: 0.00744,\tavg_G: 18,\tt: 2017,\tdt: 2.808ms,\tSimpleTD/loss: 0.00593,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00222\n",
      "INFO:TrainMonitor:ep: 1187,\tT: 2,317,369,\tG: 17,\tavg_r: 0.00947,\tavg_G: 17.9,\tt: 1795,\tdt: 2.790ms,\tSimpleTD/loss: 0.00349,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00146\n",
      "INFO:TrainMonitor:ep: 1188,\tT: 2,319,032,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.1,\tt: 1662,\tdt: 2.774ms,\tSimpleTD/loss: 0.00124,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00413\n",
      "INFO:TrainMonitor:ep: 1189,\tT: 2,321,059,\tG: 16,\tavg_r: 0.0079,\tavg_G: 17.9,\tt: 2026,\tdt: 6.334ms,\tSimpleTD/loss: 0.00586,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.000786\n",
      "INFO:TrainMonitor:ep: 1190,\tT: 2,322,724,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.1,\tt: 1664,\tdt: 2.830ms,\tSimpleTD/loss: 0.00111,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00251\n",
      "INFO:TrainMonitor:ep: 1191,\tT: 2,324,587,\tG: 18,\tavg_r: 0.00967,\tavg_G: 18.1,\tt: 1862,\tdt: 2.800ms,\tSimpleTD/loss: 0.00269,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00266\n",
      "INFO:TrainMonitor:ep: 1192,\tT: 2,326,249,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.3,\tt: 1661,\tdt: 2.825ms,\tSimpleTD/loss: 0.0011,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0028\n",
      "INFO:TrainMonitor:ep: 1193,\tT: 2,327,961,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.4,\tt: 1711,\tdt: 2.765ms,\tSimpleTD/loss: 0.00149,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00137\n",
      "INFO:TrainMonitor:ep: 1194,\tT: 2,329,777,\tG: 16,\tavg_r: 0.00882,\tavg_G: 18.1,\tt: 1815,\tdt: 2.801ms,\tSimpleTD/loss: 0.00145,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00195\n",
      "INFO:TrainMonitor:ep: 1195,\tT: 2,331,443,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.3,\tt: 1665,\tdt: 6.306ms,\tSimpleTD/loss: 0.00259,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00177\n",
      "INFO:TrainMonitor:ep: 1196,\tT: 2,333,363,\tG: 17,\tavg_r: 0.00886,\tavg_G: 18.2,\tt: 1919,\tdt: 2.795ms,\tSimpleTD/loss: 0.00291,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00336\n",
      "INFO:TrainMonitor:ep: 1197,\tT: 2,335,180,\tG: 18,\tavg_r: 0.00991,\tavg_G: 18.2,\tt: 1816,\tdt: 2.784ms,\tSimpleTD/loss: 0.00537,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.000326\n",
      "INFO:TrainMonitor:ep: 1198,\tT: 2,337,057,\tG: 18,\tavg_r: 0.00959,\tavg_G: 18.2,\tt: 1876,\tdt: 2.845ms,\tSimpleTD/loss: 0.00446,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00281\n",
      "INFO:TrainMonitor:ep: 1199,\tT: 2,338,908,\tG: 17,\tavg_r: 0.00919,\tavg_G: 18,\tt: 1850,\tdt: 2.800ms,\tSimpleTD/loss: 0.00417,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00149\n",
      "INFO:TrainMonitor:ep: 1200,\tT: 2,341,270,\tG: 12,\tavg_r: 0.00508,\tavg_G: 17.4,\tt: 2361,\tdt: 5.703ms,\tSimpleTD/loss: 0.0107,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00393\n",
      "INFO:TrainMonitor:ep: 1201,\tT: 2,343,484,\tG: 13,\tavg_r: 0.00587,\tavg_G: 17,\tt: 2213,\tdt: 2.848ms,\tSimpleTD/loss: 0.00694,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00366\n",
      "INFO:TrainMonitor:ep: 1202,\tT: 2,345,201,\tG: 19,\tavg_r: 0.0111,\tavg_G: 17.2,\tt: 1716,\tdt: 2.769ms,\tSimpleTD/loss: 0.00499,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00315\n",
      "INFO:TrainMonitor:ep: 1203,\tT: 2,346,913,\tG: 19,\tavg_r: 0.0111,\tavg_G: 17.4,\tt: 1711,\tdt: 2.813ms,\tSimpleTD/loss: 0.00289,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00277\n",
      "INFO:TrainMonitor:ep: 1204,\tT: 2,348,791,\tG: 15,\tavg_r: 0.00799,\tavg_G: 17.1,\tt: 1877,\tdt: 2.819ms,\tSimpleTD/loss: 0.00818,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00305\n",
      "INFO:TrainMonitor:ep: 1205,\tT: 2,351,164,\tG: 7,\tavg_r: 0.00295,\tavg_G: 16.1,\tt: 2372,\tdt: 5.056ms,\tSimpleTD/loss: 0.013,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00514\n",
      "INFO:TrainMonitor:ep: 1206,\tT: 2,352,823,\tG: 20,\tavg_r: 0.0121,\tavg_G: 16.5,\tt: 1658,\tdt: 2.810ms,\tSimpleTD/loss: 0.00205,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00385\n",
      "INFO:TrainMonitor:ep: 1207,\tT: 2,354,799,\tG: 15,\tavg_r: 0.00759,\tavg_G: 16.4,\tt: 1975,\tdt: 2.824ms,\tSimpleTD/loss: 0.00469,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00152\n",
      "INFO:TrainMonitor:ep: 1208,\tT: 2,356,516,\tG: 19,\tavg_r: 0.0111,\tavg_G: 16.6,\tt: 1716,\tdt: 2.814ms,\tSimpleTD/loss: 0.00189,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00353\n",
      "INFO:TrainMonitor:ep: 1209,\tT: 2,358,425,\tG: 16,\tavg_r: 0.00839,\tavg_G: 16.6,\tt: 1908,\tdt: 2.781ms,\tSimpleTD/loss: 0.0048,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00179\n",
      "INFO:TrainMonitor:ep: 1210,\tT: 2,360,422,\tG: 15,\tavg_r: 0.00752,\tavg_G: 16.4,\tt: 1996,\tdt: 5.936ms,\tSimpleTD/loss: 0.00647,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.000924\n",
      "INFO:TrainMonitor:ep: 1211,\tT: 2,362,277,\tG: 17,\tavg_r: 0.00917,\tavg_G: 16.5,\tt: 1854,\tdt: 2.819ms,\tSimpleTD/loss: 0.00635,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00401\n",
      "INFO:TrainMonitor:ep: 1212,\tT: 2,364,110,\tG: 18,\tavg_r: 0.00983,\tavg_G: 16.6,\tt: 1832,\tdt: 2.807ms,\tSimpleTD/loss: 0.00525,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.0017\n",
      "INFO:TrainMonitor:ep: 1213,\tT: 2,365,774,\tG: 20,\tavg_r: 0.012,\tavg_G: 17,\tt: 1663,\tdt: 2.835ms,\tSimpleTD/loss: 0.00408,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00283\n",
      "INFO:TrainMonitor:ep: 1214,\tT: 2,367,565,\tG: 19,\tavg_r: 0.0106,\tavg_G: 17.2,\tt: 1790,\tdt: 2.804ms,\tSimpleTD/loss: 0.00375,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00352\n",
      "INFO:TrainMonitor:ep: 1215,\tT: 2,369,252,\tG: 21,\tavg_r: 0.0125,\tavg_G: 17.5,\tt: 1686,\tdt: 2.806ms,\tSimpleTD/loss: 0.00207,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00116\n",
      "INFO:TrainMonitor:ep: 1216,\tT: 2,371,155,\tG: 17,\tavg_r: 0.00894,\tavg_G: 17.5,\tt: 1902,\tdt: 5.596ms,\tSimpleTD/loss: 0.00474,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00257\n",
      "INFO:TrainMonitor:ep: 1217,\tT: 2,372,870,\tG: 19,\tavg_r: 0.0111,\tavg_G: 17.6,\tt: 1714,\tdt: 2.822ms,\tSimpleTD/loss: 0.00188,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00425\n",
      "INFO:TrainMonitor:ep: 1218,\tT: 2,374,529,\tG: 20,\tavg_r: 0.0121,\tavg_G: 17.9,\tt: 1658,\tdt: 2.800ms,\tSimpleTD/loss: 0.000903,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00167\n",
      "INFO:TrainMonitor:ep: 1219,\tT: 2,376,258,\tG: 19,\tavg_r: 0.011,\tavg_G: 18,\tt: 1728,\tdt: 2.803ms,\tSimpleTD/loss: 0.0041,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00386\n",
      "INFO:TrainMonitor:ep: 1220,\tT: 2,378,112,\tG: 17,\tavg_r: 0.00917,\tavg_G: 17.9,\tt: 1853,\tdt: 2.780ms,\tSimpleTD/loss: 0.00451,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00158\n",
      "INFO:TrainMonitor:ep: 1221,\tT: 2,379,957,\tG: 17,\tavg_r: 0.00922,\tavg_G: 17.8,\tt: 1844,\tdt: 2.788ms,\tSimpleTD/loss: 0.00365,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00234\n",
      "INFO:TrainMonitor:ep: 1222,\tT: 2,381,675,\tG: 19,\tavg_r: 0.0111,\tavg_G: 17.9,\tt: 1717,\tdt: 5.936ms,\tSimpleTD/loss: 0.00394,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00379\n",
      "INFO:TrainMonitor:ep: 1223,\tT: 2,383,727,\tG: 17,\tavg_r: 0.00829,\tavg_G: 17.8,\tt: 2051,\tdt: 2.798ms,\tSimpleTD/loss: 0.0046,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00272\n",
      "INFO:TrainMonitor:ep: 1224,\tT: 2,385,520,\tG: 18,\tavg_r: 0.01,\tavg_G: 17.8,\tt: 1792,\tdt: 2.805ms,\tSimpleTD/loss: 0.00321,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00346\n",
      "INFO:TrainMonitor:ep: 1225,\tT: 2,387,179,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.1,\tt: 1658,\tdt: 2.779ms,\tSimpleTD/loss: 0.00121,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00246\n",
      "INFO:TrainMonitor:ep: 1226,\tT: 2,388,843,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.3,\tt: 1663,\tdt: 2.846ms,\tSimpleTD/loss: 0.000709,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00256\n",
      "INFO:TrainMonitor:ep: 1227,\tT: 2,390,560,\tG: 20,\tavg_r: 0.0117,\tavg_G: 18.4,\tt: 1716,\tdt: 5.899ms,\tSimpleTD/loss: 0.00136,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00309\n",
      "INFO:TrainMonitor:ep: 1228,\tT: 2,392,226,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.6,\tt: 1665,\tdt: 2.775ms,\tSimpleTD/loss: 0.000494,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00214\n",
      "INFO:TrainMonitor:ep: 1229,\tT: 2,393,943,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.6,\tt: 1716,\tdt: 2.826ms,\tSimpleTD/loss: 0.00208,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00302\n",
      "INFO:TrainMonitor:ep: 1230,\tT: 2,395,605,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.8,\tt: 1661,\tdt: 2.783ms,\tSimpleTD/loss: 0.000477,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00243\n",
      "INFO:TrainMonitor:ep: 1231,\tT: 2,397,607,\tG: 15,\tavg_r: 0.0075,\tavg_G: 18.4,\tt: 2001,\tdt: 2.794ms,\tSimpleTD/loss: 0.00505,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00309\n",
      "INFO:TrainMonitor:ep: 1232,\tT: 2,399,361,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18.5,\tt: 1753,\tdt: 2.814ms,\tSimpleTD/loss: 0.00209,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00371\n",
      "INFO:TrainMonitor:ep: 1233,\tT: 2,401,043,\tG: 20,\tavg_r: 0.0119,\tavg_G: 18.6,\tt: 1681,\tdt: 5.905ms,\tSimpleTD/loss: 0.000688,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00228\n",
      "INFO:TrainMonitor:ep: 1234,\tT: 2,402,702,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.7,\tt: 1658,\tdt: 2.830ms,\tSimpleTD/loss: 0.000807,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00288\n",
      "INFO:TrainMonitor:ep: 1235,\tT: 2,404,366,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.9,\tt: 1663,\tdt: 2.764ms,\tSimpleTD/loss: 0.00039,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00227\n",
      "INFO:TrainMonitor:ep: 1236,\tT: 2,406,203,\tG: 19,\tavg_r: 0.0103,\tavg_G: 18.9,\tt: 1836,\tdt: 2.832ms,\tSimpleTD/loss: 0.00135,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00278\n",
      "INFO:TrainMonitor:ep: 1237,\tT: 2,407,919,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.9,\tt: 1715,\tdt: 2.757ms,\tSimpleTD/loss: 0.0011,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00321\n",
      "INFO:TrainMonitor:ep: 1238,\tT: 2,409,637,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.9,\tt: 1717,\tdt: 2.832ms,\tSimpleTD/loss: 0.00201,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00333\n",
      "INFO:TrainMonitor:ep: 1239,\tT: 2,411,335,\tG: 19,\tavg_r: 0.0112,\tavg_G: 18.9,\tt: 1697,\tdt: 6.349ms,\tSimpleTD/loss: 0.000834,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00228\n",
      "INFO:TrainMonitor:ep: 1240,\tT: 2,413,096,\tG: 18,\tavg_r: 0.0102,\tavg_G: 18.8,\tt: 1760,\tdt: 2.808ms,\tSimpleTD/loss: 0.00255,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00237\n",
      "INFO:TrainMonitor:ep: 1241,\tT: 2,414,842,\tG: 20,\tavg_r: 0.0115,\tavg_G: 18.9,\tt: 1745,\tdt: 2.753ms,\tSimpleTD/loss: 0.00122,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00293\n",
      "INFO:TrainMonitor:ep: 1242,\tT: 2,416,503,\tG: 20,\tavg_r: 0.012,\tavg_G: 19,\tt: 1660,\tdt: 2.831ms,\tSimpleTD/loss: 0.000527,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00246\n",
      "INFO:TrainMonitor:ep: 1243,\tT: 2,418,168,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.1,\tt: 1664,\tdt: 2.795ms,\tSimpleTD/loss: 0.000477,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00258\n",
      "INFO:TrainMonitor:ep: 1244,\tT: 2,419,830,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.2,\tt: 1661,\tdt: 2.836ms,\tSimpleTD/loss: 0.000271,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00225\n",
      "INFO:TrainMonitor:ep: 1245,\tT: 2,421,528,\tG: 19,\tavg_r: 0.0112,\tavg_G: 19.2,\tt: 1697,\tdt: 6.133ms,\tSimpleTD/loss: 0.000553,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00196\n",
      "INFO:TrainMonitor:ep: 1246,\tT: 2,423,193,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.3,\tt: 1664,\tdt: 2.842ms,\tSimpleTD/loss: 0.000711,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.0026\n",
      "INFO:TrainMonitor:ep: 1247,\tT: 2,424,907,\tG: 19,\tavg_r: 0.0111,\tavg_G: 19.3,\tt: 1713,\tdt: 2.810ms,\tSimpleTD/loss: 0.00211,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00205\n",
      "INFO:TrainMonitor:ep: 1248,\tT: 2,426,573,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.3,\tt: 1665,\tdt: 2.766ms,\tSimpleTD/loss: 0.000345,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00234\n",
      "INFO:TrainMonitor:ep: 1249,\tT: 2,428,239,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.4,\tt: 1665,\tdt: 2.818ms,\tSimpleTD/loss: 0.000233,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00188\n",
      "INFO:TrainMonitor:ep: 1250,\tT: 2,430,248,\tG: 15,\tavg_r: 0.00747,\tavg_G: 19,\tt: 2008,\tdt: 5.464ms,\tSimpleTD/loss: 0.00521,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00308\n",
      "INFO:TrainMonitor:ep: 1251,\tT: 2,431,909,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.1,\tt: 1660,\tdt: 2.788ms,\tSimpleTD/loss: 0.00106,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00394\n",
      "INFO:TrainMonitor:ep: 1252,\tT: 2,433,691,\tG: 18,\tavg_r: 0.0101,\tavg_G: 19,\tt: 1781,\tdt: 2.790ms,\tSimpleTD/loss: 0.00386,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00255\n",
      "INFO:TrainMonitor:ep: 1253,\tT: 2,435,354,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.1,\tt: 1662,\tdt: 2.766ms,\tSimpleTD/loss: 0.000816,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.0031\n",
      "INFO:TrainMonitor:ep: 1254,\tT: 2,437,125,\tG: 18,\tavg_r: 0.0102,\tavg_G: 19,\tt: 1770,\tdt: 2.794ms,\tSimpleTD/loss: 0.00199,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00199\n",
      "INFO:TrainMonitor:ep: 1255,\tT: 2,438,792,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.1,\tt: 1666,\tdt: 2.827ms,\tSimpleTD/loss: 0.000471,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00268\n",
      "INFO:TrainMonitor:ep: 1256,\tT: 2,440,548,\tG: 19,\tavg_r: 0.0108,\tavg_G: 19.1,\tt: 1755,\tdt: 5.860ms,\tSimpleTD/loss: 0.00124,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00232\n",
      "INFO:TrainMonitor:ep: 1257,\tT: 2,442,374,\tG: 18,\tavg_r: 0.00986,\tavg_G: 18.9,\tt: 1825,\tdt: 2.803ms,\tSimpleTD/loss: 0.00241,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00204\n",
      "INFO:TrainMonitor:ep: 1258,\tT: 2,444,035,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.1,\tt: 1660,\tdt: 2.775ms,\tSimpleTD/loss: 0.000778,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00294\n",
      "INFO:TrainMonitor:ep: 1259,\tT: 2,445,780,\tG: 18,\tavg_r: 0.0103,\tavg_G: 18.9,\tt: 1744,\tdt: 2.814ms,\tSimpleTD/loss: 0.000742,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.0016\n",
      "INFO:TrainMonitor:ep: 1260,\tT: 2,447,442,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.1,\tt: 1661,\tdt: 2.821ms,\tSimpleTD/loss: 0.0012,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00144\n",
      "INFO:TrainMonitor:ep: 1261,\tT: 2,449,418,\tG: 17,\tavg_r: 0.00861,\tavg_G: 18.8,\tt: 1975,\tdt: 2.760ms,\tSimpleTD/loss: 0.00295,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00252\n",
      "INFO:TrainMonitor:ep: 1262,\tT: 2,451,135,\tG: 20,\tavg_r: 0.0117,\tavg_G: 19,\tt: 1716,\tdt: 6.306ms,\tSimpleTD/loss: 0.000641,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.0022\n",
      "INFO:TrainMonitor:ep: 1263,\tT: 2,453,001,\tG: 19,\tavg_r: 0.0102,\tavg_G: 19,\tt: 1865,\tdt: 2.791ms,\tSimpleTD/loss: 0.0022,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00264\n",
      "INFO:TrainMonitor:ep: 1264,\tT: 2,454,748,\tG: 19,\tavg_r: 0.0109,\tavg_G: 19,\tt: 1746,\tdt: 2.815ms,\tSimpleTD/loss: 0.000933,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00192\n",
      "INFO:TrainMonitor:ep: 1265,\tT: 2,456,648,\tG: 18,\tavg_r: 0.00948,\tavg_G: 18.9,\tt: 1899,\tdt: 2.820ms,\tSimpleTD/loss: 0.00236,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00225\n",
      "INFO:TrainMonitor:ep: 1266,\tT: 2,458,398,\tG: 18,\tavg_r: 0.0103,\tavg_G: 18.8,\tt: 1749,\tdt: 2.752ms,\tSimpleTD/loss: 0.00317,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00124\n",
      "INFO:TrainMonitor:ep: 1267,\tT: 2,460,057,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.9,\tt: 1658,\tdt: 5.966ms,\tSimpleTD/loss: 0.000652,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00288\n",
      "INFO:TrainMonitor:ep: 1268,\tT: 2,461,720,\tG: 20,\tavg_r: 0.012,\tavg_G: 19,\tt: 1662,\tdt: 2.773ms,\tSimpleTD/loss: 0.000281,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0019\n",
      "INFO:TrainMonitor:ep: 1269,\tT: 2,463,472,\tG: 19,\tavg_r: 0.0109,\tavg_G: 19,\tt: 1751,\tdt: 2.807ms,\tSimpleTD/loss: 0.00158,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00205\n",
      "INFO:TrainMonitor:ep: 1270,\tT: 2,465,182,\tG: 19,\tavg_r: 0.0111,\tavg_G: 19,\tt: 1709,\tdt: 2.808ms,\tSimpleTD/loss: 0.00167,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0022\n",
      "INFO:TrainMonitor:ep: 1271,\tT: 2,466,839,\tG: 20,\tavg_r: 0.0121,\tavg_G: 19.1,\tt: 1656,\tdt: 2.776ms,\tSimpleTD/loss: 0.000746,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00249\n",
      "INFO:TrainMonitor:ep: 1272,\tT: 2,468,677,\tG: 18,\tavg_r: 0.0098,\tavg_G: 19,\tt: 1837,\tdt: 2.828ms,\tSimpleTD/loss: 0.00186,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00217\n",
      "INFO:TrainMonitor:ep: 1273,\tT: 2,470,338,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.1,\tt: 1660,\tdt: 6.111ms,\tSimpleTD/loss: 0.000542,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00242\n",
      "INFO:TrainMonitor:ep: 1274,\tT: 2,472,150,\tG: 18,\tavg_r: 0.00994,\tavg_G: 19,\tt: 1811,\tdt: 2.783ms,\tSimpleTD/loss: 0.00341,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00205\n",
      "INFO:TrainMonitor:ep: 1275,\tT: 2,473,883,\tG: 19,\tavg_r: 0.011,\tavg_G: 19,\tt: 1732,\tdt: 2.821ms,\tSimpleTD/loss: 0.00119,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00203\n",
      "INFO:TrainMonitor:ep: 1276,\tT: 2,475,615,\tG: 19,\tavg_r: 0.011,\tavg_G: 19,\tt: 1731,\tdt: 2.822ms,\tSimpleTD/loss: 0.000818,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00184\n",
      "INFO:TrainMonitor:ep: 1277,\tT: 2,477,344,\tG: 19,\tavg_r: 0.011,\tavg_G: 19,\tt: 1728,\tdt: 2.758ms,\tSimpleTD/loss: 0.00276,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00246\n",
      "INFO:TrainMonitor:ep: 1278,\tT: 2,479,010,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.1,\tt: 1665,\tdt: 2.832ms,\tSimpleTD/loss: 0.000457,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00216\n",
      "INFO:TrainMonitor:ep: 1279,\tT: 2,480,713,\tG: 19,\tavg_r: 0.0112,\tavg_G: 19.1,\tt: 1702,\tdt: 6.003ms,\tSimpleTD/loss: 0.0012,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00169\n",
      "INFO:TrainMonitor:ep: 1280,\tT: 2,482,844,\tG: 11,\tavg_r: 0.00516,\tavg_G: 18.3,\tt: 2130,\tdt: 2.780ms,\tSimpleTD/loss: 0.00889,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00257\n",
      "INFO:TrainMonitor:ep: 1281,\tT: 2,484,540,\tG: 19,\tavg_r: 0.0112,\tavg_G: 18.3,\tt: 1695,\tdt: 2.769ms,\tSimpleTD/loss: 0.00245,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0019\n",
      "INFO:TrainMonitor:ep: 1282,\tT: 2,486,199,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.5,\tt: 1658,\tdt: 2.831ms,\tSimpleTD/loss: 0.000732,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00244\n",
      "INFO:TrainMonitor:ep: 1283,\tT: 2,487,921,\tG: 20,\tavg_r: 0.0116,\tavg_G: 18.7,\tt: 1721,\tdt: 2.802ms,\tSimpleTD/loss: 0.00191,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0024\n",
      "INFO:TrainMonitor:ep: 1284,\tT: 2,490,283,\tG: 11,\tavg_r: 0.00466,\tavg_G: 17.9,\tt: 2361,\tdt: 5.317ms,\tSimpleTD/loss: 0.00755,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00499\n",
      "INFO:TrainMonitor:ep: 1285,\tT: 2,491,950,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.1,\tt: 1666,\tdt: 2.775ms,\tSimpleTD/loss: 0.00074,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00273\n",
      "INFO:TrainMonitor:ep: 1286,\tT: 2,493,635,\tG: 20,\tavg_r: 0.0119,\tavg_G: 18.3,\tt: 1684,\tdt: 2.817ms,\tSimpleTD/loss: 0.000981,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00255\n",
      "INFO:TrainMonitor:ep: 1287,\tT: 2,495,346,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.4,\tt: 1710,\tdt: 2.827ms,\tSimpleTD/loss: 0.00122,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00201\n",
      "INFO:TrainMonitor:ep: 1288,\tT: 2,497,116,\tG: 18,\tavg_r: 0.0102,\tavg_G: 18.3,\tt: 1769,\tdt: 2.808ms,\tSimpleTD/loss: 0.00188,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00234\n",
      "INFO:TrainMonitor:ep: 1289,\tT: 2,498,745,\tG: 21,\tavg_r: 0.0129,\tavg_G: 18.6,\tt: 1628,\tdt: 2.765ms,\tSimpleTD/loss: 0.00159,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00236\n",
      "INFO:TrainMonitor:ep: 1290,\tT: 2,500,564,\tG: 17,\tavg_r: 0.00935,\tavg_G: 18.4,\tt: 1818,\tdt: 5.687ms,\tSimpleTD/loss: 0.00499,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00259\n",
      "INFO:TrainMonitor:ep: 1291,\tT: 2,502,288,\tG: 20,\tavg_r: 0.0116,\tavg_G: 18.6,\tt: 1723,\tdt: 2.827ms,\tSimpleTD/loss: 0.00188,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00401\n",
      "INFO:TrainMonitor:ep: 1292,\tT: 2,504,251,\tG: 15,\tavg_r: 0.00765,\tavg_G: 18.2,\tt: 1962,\tdt: 2.764ms,\tSimpleTD/loss: 0.00523,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00374\n",
      "INFO:TrainMonitor:ep: 1293,\tT: 2,505,978,\tG: 20,\tavg_r: 0.0116,\tavg_G: 18.4,\tt: 1726,\tdt: 2.815ms,\tSimpleTD/loss: 0.00103,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00312\n",
      "INFO:TrainMonitor:ep: 1294,\tT: 2,507,639,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.6,\tt: 1660,\tdt: 2.829ms,\tSimpleTD/loss: 0.000426,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00236\n",
      "INFO:TrainMonitor:ep: 1295,\tT: 2,509,552,\tG: 16,\tavg_r: 0.00837,\tavg_G: 18.3,\tt: 1912,\tdt: 2.798ms,\tSimpleTD/loss: 0.00439,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.0033\n",
      "INFO:TrainMonitor:ep: 1296,\tT: 2,511,468,\tG: 15,\tavg_r: 0.00783,\tavg_G: 18,\tt: 1915,\tdt: 5.734ms,\tSimpleTD/loss: 0.00497,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00194\n",
      "INFO:TrainMonitor:ep: 1297,\tT: 2,513,192,\tG: 19,\tavg_r: 0.011,\tavg_G: 18.1,\tt: 1723,\tdt: 2.789ms,\tSimpleTD/loss: 0.00218,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00296\n",
      "INFO:TrainMonitor:ep: 1298,\tT: 2,514,859,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.3,\tt: 1666,\tdt: 2.828ms,\tSimpleTD/loss: 0.000615,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0024\n",
      "INFO:TrainMonitor:ep: 1299,\tT: 2,516,607,\tG: 18,\tavg_r: 0.0103,\tavg_G: 18.2,\tt: 1747,\tdt: 2.821ms,\tSimpleTD/loss: 0.00338,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00173\n",
      "INFO:TrainMonitor:ep: 1300,\tT: 2,518,352,\tG: 20,\tavg_r: 0.0115,\tavg_G: 18.4,\tt: 1744,\tdt: 2.767ms,\tSimpleTD/loss: 0.00215,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00298\n",
      "INFO:TrainMonitor:ep: 1301,\tT: 2,520,413,\tG: 14,\tavg_r: 0.0068,\tavg_G: 18,\tt: 2060,\tdt: 5.399ms,\tSimpleTD/loss: 0.0063,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00422\n",
      "INFO:TrainMonitor:ep: 1302,\tT: 2,522,424,\tG: 13,\tavg_r: 0.00647,\tavg_G: 17.5,\tt: 2010,\tdt: 2.804ms,\tSimpleTD/loss: 0.00591,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00295\n",
      "INFO:TrainMonitor:ep: 1303,\tT: 2,524,224,\tG: 19,\tavg_r: 0.0106,\tavg_G: 17.6,\tt: 1799,\tdt: 2.810ms,\tSimpleTD/loss: 0.00224,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00176\n",
      "INFO:TrainMonitor:ep: 1304,\tT: 2,525,885,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.9,\tt: 1660,\tdt: 2.845ms,\tSimpleTD/loss: 0.00088,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00309\n",
      "INFO:TrainMonitor:ep: 1305,\tT: 2,527,547,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.1,\tt: 1661,\tdt: 2.788ms,\tSimpleTD/loss: 0.000995,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00254\n",
      "INFO:TrainMonitor:ep: 1306,\tT: 2,529,375,\tG: 18,\tavg_r: 0.00985,\tavg_G: 18.1,\tt: 1827,\tdt: 2.800ms,\tSimpleTD/loss: 0.00228,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00186\n",
      "INFO:TrainMonitor:ep: 1307,\tT: 2,531,228,\tG: 18,\tavg_r: 0.00972,\tavg_G: 18.1,\tt: 1852,\tdt: 5.904ms,\tSimpleTD/loss: 0.00336,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00175\n",
      "INFO:TrainMonitor:ep: 1308,\tT: 2,532,971,\tG: 19,\tavg_r: 0.0109,\tavg_G: 18.2,\tt: 1742,\tdt: 2.809ms,\tSimpleTD/loss: 0.00282,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00273\n",
      "INFO:TrainMonitor:ep: 1309,\tT: 2,534,908,\tG: 18,\tavg_r: 0.0093,\tavg_G: 18.1,\tt: 1936,\tdt: 2.767ms,\tSimpleTD/loss: 0.00551,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00293\n",
      "INFO:TrainMonitor:ep: 1310,\tT: 2,536,571,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.3,\tt: 1662,\tdt: 2.834ms,\tSimpleTD/loss: 0.00083,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00188\n",
      "INFO:TrainMonitor:ep: 1311,\tT: 2,538,382,\tG: 18,\tavg_r: 0.00994,\tavg_G: 18.3,\tt: 1810,\tdt: 2.806ms,\tSimpleTD/loss: 0.00281,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00292\n",
      "INFO:TrainMonitor:ep: 1312,\tT: 2,540,103,\tG: 20,\tavg_r: 0.0116,\tavg_G: 18.5,\tt: 1720,\tdt: 6.198ms,\tSimpleTD/loss: 0.00132,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00303\n",
      "INFO:TrainMonitor:ep: 1313,\tT: 2,542,037,\tG: 16,\tavg_r: 0.00828,\tavg_G: 18.2,\tt: 1933,\tdt: 2.807ms,\tSimpleTD/loss: 0.00465,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00249\n",
      "INFO:TrainMonitor:ep: 1314,\tT: 2,543,695,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.4,\tt: 1657,\tdt: 2.788ms,\tSimpleTD/loss: 0.000745,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00293\n",
      "INFO:TrainMonitor:ep: 1315,\tT: 2,545,354,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.6,\tt: 1658,\tdt: 2.839ms,\tSimpleTD/loss: 0.000498,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00182\n",
      "INFO:TrainMonitor:ep: 1316,\tT: 2,547,300,\tG: 16,\tavg_r: 0.00823,\tavg_G: 18.3,\tt: 1945,\tdt: 2.778ms,\tSimpleTD/loss: 0.00372,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00398\n",
      "INFO:TrainMonitor:ep: 1317,\tT: 2,549,231,\tG: 16,\tavg_r: 0.00829,\tavg_G: 18.1,\tt: 1930,\tdt: 2.839ms,\tSimpleTD/loss: 0.00281,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00212\n",
      "INFO:TrainMonitor:ep: 1318,\tT: 2,551,388,\tG: 10,\tavg_r: 0.00464,\tavg_G: 17.3,\tt: 2156,\tdt: 5.287ms,\tSimpleTD/loss: 0.00762,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00334\n",
      "INFO:TrainMonitor:ep: 1319,\tT: 2,553,054,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.5,\tt: 1665,\tdt: 2.825ms,\tSimpleTD/loss: 0.000828,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00224\n",
      "INFO:TrainMonitor:ep: 1320,\tT: 2,554,941,\tG: 16,\tavg_r: 0.00848,\tavg_G: 17.4,\tt: 1886,\tdt: 2.780ms,\tSimpleTD/loss: 0.00656,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00235\n",
      "INFO:TrainMonitor:ep: 1321,\tT: 2,556,609,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.6,\tt: 1667,\tdt: 2.829ms,\tSimpleTD/loss: 0.000814,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00298\n",
      "INFO:TrainMonitor:ep: 1322,\tT: 2,558,429,\tG: 16,\tavg_r: 0.0088,\tavg_G: 17.5,\tt: 1819,\tdt: 2.788ms,\tSimpleTD/loss: 0.00384,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00175\n",
      "INFO:TrainMonitor:ep: 1323,\tT: 2,560,074,\tG: 21,\tavg_r: 0.0128,\tavg_G: 17.8,\tt: 1644,\tdt: 5.990ms,\tSimpleTD/loss: 0.00201,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00216\n",
      "INFO:TrainMonitor:ep: 1324,\tT: 2,561,824,\tG: 19,\tavg_r: 0.0109,\tavg_G: 18,\tt: 1749,\tdt: 2.860ms,\tSimpleTD/loss: 0.00195,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00364\n",
      "INFO:TrainMonitor:ep: 1325,\tT: 2,563,562,\tG: 19,\tavg_r: 0.0109,\tavg_G: 18.1,\tt: 1737,\tdt: 2.816ms,\tSimpleTD/loss: 0.00179,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0017\n",
      "INFO:TrainMonitor:ep: 1326,\tT: 2,565,220,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.2,\tt: 1657,\tdt: 2.788ms,\tSimpleTD/loss: 0.0012,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00363\n",
      "INFO:TrainMonitor:ep: 1327,\tT: 2,566,887,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.4,\tt: 1666,\tdt: 2.835ms,\tSimpleTD/loss: 0.000562,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00238\n",
      "INFO:TrainMonitor:ep: 1328,\tT: 2,568,654,\tG: 18,\tavg_r: 0.0102,\tavg_G: 18.4,\tt: 1766,\tdt: 2.803ms,\tSimpleTD/loss: 0.00417,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.0011\n",
      "INFO:TrainMonitor:ep: 1329,\tT: 2,570,321,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.5,\tt: 1666,\tdt: 5.955ms,\tSimpleTD/loss: 0.00104,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00264\n",
      "INFO:TrainMonitor:ep: 1330,\tT: 2,572,214,\tG: 19,\tavg_r: 0.01,\tavg_G: 18.6,\tt: 1892,\tdt: 2.834ms,\tSimpleTD/loss: 0.0026,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00286\n",
      "INFO:TrainMonitor:ep: 1331,\tT: 2,573,876,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.7,\tt: 1661,\tdt: 2.783ms,\tSimpleTD/loss: 0.000712,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00316\n",
      "INFO:TrainMonitor:ep: 1332,\tT: 2,575,537,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.9,\tt: 1660,\tdt: 2.851ms,\tSimpleTD/loss: 0.000482,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00261\n",
      "INFO:TrainMonitor:ep: 1333,\tT: 2,577,348,\tG: 17,\tavg_r: 0.00939,\tavg_G: 18.7,\tt: 1810,\tdt: 2.802ms,\tSimpleTD/loss: 0.00349,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00315\n",
      "INFO:TrainMonitor:ep: 1334,\tT: 2,579,016,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.8,\tt: 1667,\tdt: 2.770ms,\tSimpleTD/loss: 0.00168,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00298\n",
      "INFO:TrainMonitor:ep: 1335,\tT: 2,580,813,\tG: 18,\tavg_r: 0.01,\tavg_G: 18.7,\tt: 1796,\tdt: 5.725ms,\tSimpleTD/loss: 0.00341,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00176\n",
      "INFO:TrainMonitor:ep: 1336,\tT: 2,582,505,\tG: 21,\tavg_r: 0.0124,\tavg_G: 19,\tt: 1691,\tdt: 2.819ms,\tSimpleTD/loss: 0.000967,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00157\n",
      "INFO:TrainMonitor:ep: 1337,\tT: 2,584,345,\tG: 19,\tavg_r: 0.0103,\tavg_G: 19,\tt: 1839,\tdt: 2.790ms,\tSimpleTD/loss: 0.00416,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00404\n",
      "INFO:TrainMonitor:ep: 1338,\tT: 2,586,123,\tG: 19,\tavg_r: 0.0107,\tavg_G: 19,\tt: 1777,\tdt: 2.819ms,\tSimpleTD/loss: 0.00163,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00315\n",
      "INFO:TrainMonitor:ep: 1339,\tT: 2,587,787,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.1,\tt: 1663,\tdt: 2.826ms,\tSimpleTD/loss: 0.000521,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00186\n",
      "INFO:TrainMonitor:ep: 1340,\tT: 2,589,513,\tG: 20,\tavg_r: 0.0116,\tavg_G: 19.2,\tt: 1725,\tdt: 2.775ms,\tSimpleTD/loss: 0.0016,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00342\n",
      "INFO:TrainMonitor:ep: 1341,\tT: 2,591,211,\tG: 19,\tavg_r: 0.0112,\tavg_G: 19.1,\tt: 1697,\tdt: 6.069ms,\tSimpleTD/loss: 0.00154,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.000989\n",
      "INFO:TrainMonitor:ep: 1342,\tT: 2,592,949,\tG: 18,\tavg_r: 0.0104,\tavg_G: 19,\tt: 1737,\tdt: 2.828ms,\tSimpleTD/loss: 0.00294,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.002\n",
      "INFO:TrainMonitor:ep: 1343,\tT: 2,594,807,\tG: 17,\tavg_r: 0.00915,\tavg_G: 18.8,\tt: 1857,\tdt: 2.796ms,\tSimpleTD/loss: 0.00506,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00398\n",
      "INFO:TrainMonitor:ep: 1344,\tT: 2,596,520,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.8,\tt: 1712,\tdt: 2.818ms,\tSimpleTD/loss: 0.00225,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00324\n",
      "INFO:TrainMonitor:ep: 1345,\tT: 2,598,214,\tG: 20,\tavg_r: 0.0118,\tavg_G: 19,\tt: 1693,\tdt: 2.789ms,\tSimpleTD/loss: 0.000973,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0022\n",
      "INFO:TrainMonitor:ep: 1346,\tT: 2,599,971,\tG: 19,\tavg_r: 0.0108,\tavg_G: 19,\tt: 1756,\tdt: 2.808ms,\tSimpleTD/loss: 0.00135,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00252\n",
      "INFO:TrainMonitor:ep: 1347,\tT: 2,601,723,\tG: 18,\tavg_r: 0.0103,\tavg_G: 18.9,\tt: 1751,\tdt: 5.922ms,\tSimpleTD/loss: 0.0042,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00212\n",
      "INFO:TrainMonitor:ep: 1348,\tT: 2,603,385,\tG: 20,\tavg_r: 0.012,\tavg_G: 19,\tt: 1661,\tdt: 2.782ms,\tSimpleTD/loss: 0.000681,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0032\n",
      "INFO:TrainMonitor:ep: 1349,\tT: 2,605,152,\tG: 18,\tavg_r: 0.0102,\tavg_G: 18.9,\tt: 1766,\tdt: 2.815ms,\tSimpleTD/loss: 0.00233,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00174\n",
      "INFO:TrainMonitor:ep: 1350,\tT: 2,606,780,\tG: 21,\tavg_r: 0.0129,\tavg_G: 19.1,\tt: 1627,\tdt: 2.844ms,\tSimpleTD/loss: 0.00112,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00349\n",
      "INFO:TrainMonitor:ep: 1351,\tT: 2,608,438,\tG: 20,\tavg_r: 0.0121,\tavg_G: 19.2,\tt: 1657,\tdt: 2.796ms,\tSimpleTD/loss: 0.00266,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.0034\n",
      "INFO:TrainMonitor:ep: 1352,\tT: 2,610,115,\tG: 20,\tavg_r: 0.0119,\tavg_G: 19.3,\tt: 1676,\tdt: 6.025ms,\tSimpleTD/loss: 0.000915,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00266\n",
      "INFO:TrainMonitor:ep: 1353,\tT: 2,611,869,\tG: 18,\tavg_r: 0.0103,\tavg_G: 19.1,\tt: 1753,\tdt: 2.819ms,\tSimpleTD/loss: 0.00239,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00134\n",
      "INFO:TrainMonitor:ep: 1354,\tT: 2,613,527,\tG: 20,\tavg_r: 0.0121,\tavg_G: 19.2,\tt: 1657,\tdt: 2.785ms,\tSimpleTD/loss: 0.000916,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00302\n",
      "INFO:TrainMonitor:ep: 1355,\tT: 2,615,645,\tG: 12,\tavg_r: 0.00567,\tavg_G: 18.5,\tt: 2117,\tdt: 2.793ms,\tSimpleTD/loss: 0.00932,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00495\n",
      "INFO:TrainMonitor:ep: 1356,\tT: 2,617,343,\tG: 20,\tavg_r: 0.0118,\tavg_G: 18.7,\tt: 1697,\tdt: 2.827ms,\tSimpleTD/loss: 0.001,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00268\n",
      "INFO:TrainMonitor:ep: 1357,\tT: 2,619,629,\tG: 14,\tavg_r: 0.00613,\tavg_G: 18.2,\tt: 2285,\tdt: 2.802ms,\tSimpleTD/loss: 0.00559,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00449\n",
      "INFO:TrainMonitor:ep: 1358,\tT: 2,621,465,\tG: 19,\tavg_r: 0.0104,\tavg_G: 18.3,\tt: 1835,\tdt: 5.702ms,\tSimpleTD/loss: 0.00323,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00196\n",
      "INFO:TrainMonitor:ep: 1359,\tT: 2,623,123,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.4,\tt: 1657,\tdt: 2.786ms,\tSimpleTD/loss: 0.00071,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00307\n",
      "INFO:TrainMonitor:ep: 1360,\tT: 2,624,784,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.6,\tt: 1660,\tdt: 2.818ms,\tSimpleTD/loss: 0.000382,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00215\n",
      "INFO:TrainMonitor:ep: 1361,\tT: 2,626,841,\tG: 16,\tavg_r: 0.00778,\tavg_G: 18.3,\tt: 2056,\tdt: 2.822ms,\tSimpleTD/loss: 0.00459,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00266\n",
      "INFO:TrainMonitor:ep: 1362,\tT: 2,628,616,\tG: 18,\tavg_r: 0.0101,\tavg_G: 18.3,\tt: 1774,\tdt: 2.805ms,\tSimpleTD/loss: 0.00244,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00298\n",
      "INFO:TrainMonitor:ep: 1363,\tT: 2,630,388,\tG: 19,\tavg_r: 0.0107,\tavg_G: 18.4,\tt: 1771,\tdt: 5.755ms,\tSimpleTD/loss: 0.0028,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00335\n",
      "INFO:TrainMonitor:ep: 1364,\tT: 2,632,223,\tG: 17,\tavg_r: 0.00927,\tavg_G: 18.2,\tt: 1834,\tdt: 2.796ms,\tSimpleTD/loss: 0.00318,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00156\n",
      "INFO:TrainMonitor:ep: 1365,\tT: 2,633,885,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.4,\tt: 1661,\tdt: 2.777ms,\tSimpleTD/loss: 0.000698,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00254\n",
      "INFO:TrainMonitor:ep: 1366,\tT: 2,635,698,\tG: 19,\tavg_r: 0.0105,\tavg_G: 18.5,\tt: 1812,\tdt: 2.844ms,\tSimpleTD/loss: 0.00141,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00216\n",
      "INFO:TrainMonitor:ep: 1367,\tT: 2,637,471,\tG: 18,\tavg_r: 0.0102,\tavg_G: 18.4,\tt: 1772,\tdt: 2.778ms,\tSimpleTD/loss: 0.00256,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00182\n",
      "INFO:TrainMonitor:ep: 1368,\tT: 2,639,313,\tG: 17,\tavg_r: 0.00923,\tavg_G: 18.3,\tt: 1841,\tdt: 2.851ms,\tSimpleTD/loss: 0.00327,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00287\n",
      "INFO:TrainMonitor:ep: 1369,\tT: 2,641,036,\tG: 20,\tavg_r: 0.0116,\tavg_G: 18.5,\tt: 1722,\tdt: 6.036ms,\tSimpleTD/loss: 0.00111,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00379\n",
      "INFO:TrainMonitor:ep: 1370,\tT: 2,642,839,\tG: 17,\tavg_r: 0.00943,\tavg_G: 18.3,\tt: 1802,\tdt: 2.796ms,\tSimpleTD/loss: 0.00349,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00183\n",
      "INFO:TrainMonitor:ep: 1371,\tT: 2,644,583,\tG: 20,\tavg_r: 0.0115,\tavg_G: 18.5,\tt: 1743,\tdt: 2.808ms,\tSimpleTD/loss: 0.00204,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00224\n",
      "INFO:TrainMonitor:ep: 1372,\tT: 2,646,419,\tG: 17,\tavg_r: 0.00926,\tavg_G: 18.3,\tt: 1835,\tdt: 2.788ms,\tSimpleTD/loss: 0.00363,\tPPOClip/EntropyRegularizer/entropy: 1.53,\tPPOClip/loss: -0.00379\n",
      "INFO:TrainMonitor:ep: 1373,\tT: 2,648,094,\tG: 20,\tavg_r: 0.0119,\tavg_G: 18.5,\tt: 1674,\tdt: 2.820ms,\tSimpleTD/loss: 0.00136,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00241\n",
      "INFO:TrainMonitor:ep: 1374,\tT: 2,649,755,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.6,\tt: 1660,\tdt: 2.765ms,\tSimpleTD/loss: 0.000618,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00216\n",
      "INFO:TrainMonitor:ep: 1375,\tT: 2,651,419,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.8,\tt: 1663,\tdt: 6.088ms,\tSimpleTD/loss: 0.000291,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00205\n",
      "INFO:TrainMonitor:ep: 1376,\tT: 2,653,197,\tG: 19,\tavg_r: 0.0107,\tavg_G: 18.8,\tt: 1777,\tdt: 2.817ms,\tSimpleTD/loss: 0.00154,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00296\n",
      "INFO:TrainMonitor:ep: 1377,\tT: 2,654,991,\tG: 18,\tavg_r: 0.01,\tavg_G: 18.7,\tt: 1793,\tdt: 2.801ms,\tSimpleTD/loss: 0.00193,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.0018\n",
      "INFO:TrainMonitor:ep: 1378,\tT: 2,656,862,\tG: 17,\tavg_r: 0.00909,\tavg_G: 18.6,\tt: 1870,\tdt: 2.792ms,\tSimpleTD/loss: 0.0027,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0023\n",
      "INFO:TrainMonitor:ep: 1379,\tT: 2,658,562,\tG: 20,\tavg_r: 0.0118,\tavg_G: 18.7,\tt: 1699,\tdt: 2.823ms,\tSimpleTD/loss: 0.000767,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00229\n",
      "INFO:TrainMonitor:ep: 1380,\tT: 2,660,225,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.8,\tt: 1662,\tdt: 6.111ms,\tSimpleTD/loss: 0.00035,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00261\n",
      "INFO:TrainMonitor:ep: 1381,\tT: 2,662,041,\tG: 18,\tavg_r: 0.00992,\tavg_G: 18.7,\tt: 1815,\tdt: 2.797ms,\tSimpleTD/loss: 0.00372,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00267\n",
      "INFO:TrainMonitor:ep: 1382,\tT: 2,663,782,\tG: 20,\tavg_r: 0.0115,\tavg_G: 18.9,\tt: 1740,\tdt: 2.820ms,\tSimpleTD/loss: 0.00137,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00242\n",
      "INFO:TrainMonitor:ep: 1383,\tT: 2,665,508,\tG: 20,\tavg_r: 0.0116,\tavg_G: 19,\tt: 1725,\tdt: 2.833ms,\tSimpleTD/loss: 0.00141,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00288\n",
      "INFO:TrainMonitor:ep: 1384,\tT: 2,667,320,\tG: 18,\tavg_r: 0.00994,\tavg_G: 18.9,\tt: 1811,\tdt: 2.799ms,\tSimpleTD/loss: 0.00206,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00176\n",
      "INFO:TrainMonitor:ep: 1385,\tT: 2,669,105,\tG: 19,\tavg_r: 0.0107,\tavg_G: 18.9,\tt: 1784,\tdt: 2.808ms,\tSimpleTD/loss: 0.00138,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00258\n",
      "INFO:TrainMonitor:ep: 1386,\tT: 2,670,797,\tG: 21,\tavg_r: 0.0124,\tavg_G: 19.1,\tt: 1691,\tdt: 6.005ms,\tSimpleTD/loss: 0.00105,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00224\n",
      "INFO:TrainMonitor:ep: 1387,\tT: 2,672,574,\tG: 19,\tavg_r: 0.0107,\tavg_G: 19.1,\tt: 1776,\tdt: 2.765ms,\tSimpleTD/loss: 0.00316,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00325\n",
      "INFO:TrainMonitor:ep: 1388,\tT: 2,674,293,\tG: 19,\tavg_r: 0.0111,\tavg_G: 19.1,\tt: 1718,\tdt: 2.822ms,\tSimpleTD/loss: 0.00185,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0017\n",
      "INFO:TrainMonitor:ep: 1389,\tT: 2,676,093,\tG: 20,\tavg_r: 0.0111,\tavg_G: 19.2,\tt: 1799,\tdt: 2.806ms,\tSimpleTD/loss: 0.0014,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00167\n",
      "INFO:TrainMonitor:ep: 1390,\tT: 2,677,814,\tG: 19,\tavg_r: 0.011,\tavg_G: 19.2,\tt: 1720,\tdt: 2.812ms,\tSimpleTD/loss: 0.00127,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00296\n",
      "INFO:TrainMonitor:ep: 1391,\tT: 2,679,541,\tG: 19,\tavg_r: 0.011,\tavg_G: 19.1,\tt: 1726,\tdt: 2.828ms,\tSimpleTD/loss: 0.00128,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00221\n",
      "INFO:TrainMonitor:ep: 1392,\tT: 2,681,309,\tG: 18,\tavg_r: 0.0102,\tavg_G: 19,\tt: 1767,\tdt: 5.781ms,\tSimpleTD/loss: 0.00365,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.0023\n",
      "INFO:TrainMonitor:ep: 1393,\tT: 2,682,966,\tG: 20,\tavg_r: 0.0121,\tavg_G: 19.1,\tt: 1656,\tdt: 2.777ms,\tSimpleTD/loss: 0.00131,\tPPOClip/EntropyRegularizer/entropy: 1.54,\tPPOClip/loss: -0.00289\n",
      "INFO:TrainMonitor:ep: 1394,\tT: 2,684,625,\tG: 20,\tavg_r: 0.0121,\tavg_G: 19.2,\tt: 1658,\tdt: 2.820ms,\tSimpleTD/loss: 0.000501,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00214\n",
      "INFO:TrainMonitor:ep: 1395,\tT: 2,686,290,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.3,\tt: 1664,\tdt: 2.783ms,\tSimpleTD/loss: 0.000433,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00218\n",
      "INFO:TrainMonitor:ep: 1396,\tT: 2,688,086,\tG: 18,\tavg_r: 0.01,\tavg_G: 19.2,\tt: 1795,\tdt: 2.837ms,\tSimpleTD/loss: 0.00249,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00259\n",
      "INFO:TrainMonitor:ep: 1397,\tT: 2,689,838,\tG: 19,\tavg_r: 0.0109,\tavg_G: 19.1,\tt: 1751,\tdt: 2.815ms,\tSimpleTD/loss: 0.00146,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00255\n",
      "INFO:TrainMonitor:ep: 1398,\tT: 2,691,573,\tG: 19,\tavg_r: 0.011,\tavg_G: 19.1,\tt: 1734,\tdt: 5.874ms,\tSimpleTD/loss: 0.00276,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0025\n",
      "INFO:TrainMonitor:ep: 1399,\tT: 2,693,361,\tG: 18,\tavg_r: 0.0101,\tavg_G: 19,\tt: 1787,\tdt: 2.825ms,\tSimpleTD/loss: 0.00311,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00132\n",
      "INFO:TrainMonitor:ep: 1400,\tT: 2,695,115,\tG: 19,\tavg_r: 0.0108,\tavg_G: 19,\tt: 1753,\tdt: 2.761ms,\tSimpleTD/loss: 0.00143,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00242\n",
      "INFO:TrainMonitor:ep: 1401,\tT: 2,696,801,\tG: 20,\tavg_r: 0.0119,\tavg_G: 19.1,\tt: 1685,\tdt: 2.820ms,\tSimpleTD/loss: 0.00146,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00292\n",
      "INFO:TrainMonitor:ep: 1402,\tT: 2,698,549,\tG: 20,\tavg_r: 0.0114,\tavg_G: 19.2,\tt: 1747,\tdt: 2.831ms,\tSimpleTD/loss: 0.00117,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00193\n",
      "INFO:TrainMonitor:ep: 1403,\tT: 2,700,233,\tG: 20,\tavg_r: 0.0119,\tavg_G: 19.3,\tt: 1683,\tdt: 5.933ms,\tSimpleTD/loss: 0.00206,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00274\n",
      "INFO:TrainMonitor:ep: 1404,\tT: 2,701,897,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.4,\tt: 1663,\tdt: 2.835ms,\tSimpleTD/loss: 0.00043,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00208\n",
      "INFO:TrainMonitor:ep: 1405,\tT: 2,703,542,\tG: 21,\tavg_r: 0.0128,\tavg_G: 19.5,\tt: 1644,\tdt: 2.789ms,\tSimpleTD/loss: 0.000845,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.0017\n",
      "INFO:TrainMonitor:ep: 1406,\tT: 2,705,203,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.6,\tt: 1660,\tdt: 2.841ms,\tSimpleTD/loss: 0.00047,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0023\n",
      "INFO:TrainMonitor:ep: 1407,\tT: 2,706,861,\tG: 20,\tavg_r: 0.0121,\tavg_G: 19.6,\tt: 1657,\tdt: 2.772ms,\tSimpleTD/loss: 0.00126,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00212\n",
      "INFO:TrainMonitor:ep: 1408,\tT: 2,708,527,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.6,\tt: 1665,\tdt: 2.847ms,\tSimpleTD/loss: 0.000375,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00214\n",
      "INFO:TrainMonitor:ep: 1409,\tT: 2,710,184,\tG: 20,\tavg_r: 0.0121,\tavg_G: 19.7,\tt: 1656,\tdt: 5.991ms,\tSimpleTD/loss: 0.000294,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00223\n",
      "INFO:TrainMonitor:ep: 1410,\tT: 2,712,001,\tG: 18,\tavg_r: 0.00991,\tavg_G: 19.5,\tt: 1816,\tdt: 2.811ms,\tSimpleTD/loss: 0.00283,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00274\n",
      "INFO:TrainMonitor:ep: 1411,\tT: 2,714,126,\tG: 11,\tavg_r: 0.00518,\tavg_G: 18.7,\tt: 2124,\tdt: 2.829ms,\tSimpleTD/loss: 0.00568,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00276\n",
      "INFO:TrainMonitor:ep: 1412,\tT: 2,715,773,\tG: 21,\tavg_r: 0.0128,\tavg_G: 18.9,\tt: 1646,\tdt: 2.776ms,\tSimpleTD/loss: 0.00112,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00234\n",
      "INFO:TrainMonitor:ep: 1413,\tT: 2,717,546,\tG: 19,\tavg_r: 0.0107,\tavg_G: 18.9,\tt: 1772,\tdt: 2.806ms,\tSimpleTD/loss: 0.00192,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00251\n",
      "INFO:TrainMonitor:ep: 1414,\tT: 2,719,212,\tG: 20,\tavg_r: 0.012,\tavg_G: 19,\tt: 1665,\tdt: 2.822ms,\tSimpleTD/loss: 0.00234,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00111\n",
      "INFO:TrainMonitor:ep: 1415,\tT: 2,720,874,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.1,\tt: 1661,\tdt: 6.124ms,\tSimpleTD/loss: 0.000752,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00241\n",
      "INFO:TrainMonitor:ep: 1416,\tT: 2,722,526,\tG: 21,\tavg_r: 0.0127,\tavg_G: 19.3,\tt: 1651,\tdt: 2.832ms,\tSimpleTD/loss: 0.000906,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00168\n",
      "INFO:TrainMonitor:ep: 1417,\tT: 2,724,192,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.4,\tt: 1665,\tdt: 2.784ms,\tSimpleTD/loss: 0.000469,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00191\n",
      "INFO:TrainMonitor:ep: 1418,\tT: 2,725,855,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.4,\tt: 1662,\tdt: 2.782ms,\tSimpleTD/loss: 0.000323,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00177\n",
      "INFO:TrainMonitor:ep: 1419,\tT: 2,727,578,\tG: 19,\tavg_r: 0.011,\tavg_G: 19.4,\tt: 1722,\tdt: 2.805ms,\tSimpleTD/loss: 0.000292,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00211\n",
      "INFO:TrainMonitor:ep: 1420,\tT: 2,729,764,\tG: 13,\tavg_r: 0.00595,\tavg_G: 18.8,\tt: 2185,\tdt: 2.831ms,\tSimpleTD/loss: 0.00703,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.0031\n",
      "INFO:TrainMonitor:ep: 1421,\tT: 2,731,715,\tG: 15,\tavg_r: 0.00769,\tavg_G: 18.4,\tt: 1950,\tdt: 5.647ms,\tSimpleTD/loss: 0.00759,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.0045\n",
      "INFO:TrainMonitor:ep: 1422,\tT: 2,733,664,\tG: 16,\tavg_r: 0.00821,\tavg_G: 18.1,\tt: 1948,\tdt: 2.817ms,\tSimpleTD/loss: 0.00653,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00326\n",
      "INFO:TrainMonitor:ep: 1423,\tT: 2,735,491,\tG: 18,\tavg_r: 0.00986,\tavg_G: 18.1,\tt: 1826,\tdt: 2.819ms,\tSimpleTD/loss: 0.00616,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00323\n",
      "INFO:TrainMonitor:ep: 1424,\tT: 2,737,187,\tG: 19,\tavg_r: 0.0112,\tavg_G: 18.2,\tt: 1695,\tdt: 2.847ms,\tSimpleTD/loss: 0.00125,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00211\n",
      "INFO:TrainMonitor:ep: 1425,\tT: 2,739,128,\tG: 17,\tavg_r: 0.00876,\tavg_G: 18.1,\tt: 1940,\tdt: 2.785ms,\tSimpleTD/loss: 0.0057,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00398\n",
      "INFO:TrainMonitor:ep: 1426,\tT: 2,741,501,\tG: 10,\tavg_r: 0.00422,\tavg_G: 17.3,\tt: 2372,\tdt: 5.407ms,\tSimpleTD/loss: 0.00937,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00465\n",
      "INFO:TrainMonitor:ep: 1427,\tT: 2,743,205,\tG: 20,\tavg_r: 0.0117,\tavg_G: 17.6,\tt: 1703,\tdt: 2.771ms,\tSimpleTD/loss: 0.00453,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00422\n",
      "INFO:TrainMonitor:ep: 1428,\tT: 2,744,987,\tG: 19,\tavg_r: 0.0107,\tavg_G: 17.7,\tt: 1781,\tdt: 2.805ms,\tSimpleTD/loss: 0.0027,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00211\n",
      "INFO:TrainMonitor:ep: 1429,\tT: 2,746,695,\tG: 20,\tavg_r: 0.0117,\tavg_G: 17.9,\tt: 1707,\tdt: 2.807ms,\tSimpleTD/loss: 0.00235,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00287\n",
      "INFO:TrainMonitor:ep: 1430,\tT: 2,748,459,\tG: 20,\tavg_r: 0.0113,\tavg_G: 18.1,\tt: 1763,\tdt: 2.804ms,\tSimpleTD/loss: 0.00325,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.0037\n",
      "INFO:TrainMonitor:ep: 1431,\tT: 2,750,205,\tG: 20,\tavg_r: 0.0115,\tavg_G: 18.3,\tt: 1745,\tdt: 6.027ms,\tSimpleTD/loss: 0.00205,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00319\n",
      "INFO:TrainMonitor:ep: 1432,\tT: 2,752,218,\tG: 15,\tavg_r: 0.00746,\tavg_G: 18,\tt: 2012,\tdt: 2.771ms,\tSimpleTD/loss: 0.00361,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00255\n",
      "INFO:TrainMonitor:ep: 1433,\tT: 2,753,882,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.2,\tt: 1663,\tdt: 2.813ms,\tSimpleTD/loss: 0.00613,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00481\n",
      "INFO:TrainMonitor:ep: 1434,\tT: 2,755,644,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18.3,\tt: 1761,\tdt: 2.816ms,\tSimpleTD/loss: 0.00139,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00255\n",
      "INFO:TrainMonitor:ep: 1435,\tT: 2,757,449,\tG: 19,\tavg_r: 0.0105,\tavg_G: 18.3,\tt: 1804,\tdt: 2.797ms,\tSimpleTD/loss: 0.00187,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00278\n",
      "INFO:TrainMonitor:ep: 1436,\tT: 2,759,117,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.5,\tt: 1667,\tdt: 2.763ms,\tSimpleTD/loss: 0.000646,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00253\n",
      "INFO:TrainMonitor:ep: 1437,\tT: 2,760,832,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.6,\tt: 1714,\tdt: 6.179ms,\tSimpleTD/loss: 0.000425,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00209\n",
      "INFO:TrainMonitor:ep: 1438,\tT: 2,762,493,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.7,\tt: 1660,\tdt: 2.824ms,\tSimpleTD/loss: 0.00116,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00218\n",
      "INFO:TrainMonitor:ep: 1439,\tT: 2,764,246,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18.7,\tt: 1752,\tdt: 2.768ms,\tSimpleTD/loss: 0.000916,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00231\n",
      "INFO:TrainMonitor:ep: 1440,\tT: 2,765,940,\tG: 21,\tavg_r: 0.0124,\tavg_G: 19,\tt: 1693,\tdt: 2.816ms,\tSimpleTD/loss: 0.00057,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00209\n",
      "INFO:TrainMonitor:ep: 1441,\tT: 2,767,842,\tG: 17,\tavg_r: 0.00894,\tavg_G: 18.8,\tt: 1901,\tdt: 2.774ms,\tSimpleTD/loss: 0.00359,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00214\n",
      "INFO:TrainMonitor:ep: 1442,\tT: 2,769,689,\tG: 17,\tavg_r: 0.00921,\tavg_G: 18.6,\tt: 1846,\tdt: 2.834ms,\tSimpleTD/loss: 0.00292,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00258\n",
      "INFO:TrainMonitor:ep: 1443,\tT: 2,771,435,\tG: 18,\tavg_r: 0.0103,\tavg_G: 18.5,\tt: 1745,\tdt: 5.765ms,\tSimpleTD/loss: 0.00371,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00186\n",
      "INFO:TrainMonitor:ep: 1444,\tT: 2,773,330,\tG: 16,\tavg_r: 0.00845,\tavg_G: 18.3,\tt: 1894,\tdt: 2.833ms,\tSimpleTD/loss: 0.00509,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00456\n",
      "INFO:TrainMonitor:ep: 1445,\tT: 2,774,993,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.4,\tt: 1662,\tdt: 2.774ms,\tSimpleTD/loss: 0.000834,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00301\n",
      "INFO:TrainMonitor:ep: 1446,\tT: 2,776,719,\tG: 20,\tavg_r: 0.0116,\tavg_G: 18.6,\tt: 1725,\tdt: 2.815ms,\tSimpleTD/loss: 0.00109,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00252\n",
      "INFO:TrainMonitor:ep: 1447,\tT: 2,778,385,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.7,\tt: 1665,\tdt: 2.842ms,\tSimpleTD/loss: 0.000697,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00233\n",
      "INFO:TrainMonitor:ep: 1448,\tT: 2,780,042,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.9,\tt: 1656,\tdt: 6.245ms,\tSimpleTD/loss: 0.000417,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00209\n",
      "INFO:TrainMonitor:ep: 1449,\tT: 2,781,756,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.9,\tt: 1713,\tdt: 2.803ms,\tSimpleTD/loss: 0.00211,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00248\n",
      "INFO:TrainMonitor:ep: 1450,\tT: 2,783,555,\tG: 18,\tavg_r: 0.01,\tavg_G: 18.8,\tt: 1798,\tdt: 2.792ms,\tSimpleTD/loss: 0.00245,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00309\n",
      "INFO:TrainMonitor:ep: 1451,\tT: 2,785,398,\tG: 18,\tavg_r: 0.00977,\tavg_G: 18.7,\tt: 1842,\tdt: 2.785ms,\tSimpleTD/loss: 0.00301,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00206\n",
      "INFO:TrainMonitor:ep: 1452,\tT: 2,787,430,\tG: 17,\tavg_r: 0.00837,\tavg_G: 18.5,\tt: 2031,\tdt: 2.801ms,\tSimpleTD/loss: 0.00358,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00243\n",
      "INFO:TrainMonitor:ep: 1453,\tT: 2,789,164,\tG: 19,\tavg_r: 0.011,\tavg_G: 18.6,\tt: 1733,\tdt: 2.813ms,\tSimpleTD/loss: 0.00177,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00201\n",
      "INFO:TrainMonitor:ep: 1454,\tT: 2,791,281,\tG: 14,\tavg_r: 0.00662,\tavg_G: 18.1,\tt: 2116,\tdt: 5.968ms,\tSimpleTD/loss: 0.00473,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.0037\n",
      "INFO:TrainMonitor:ep: 1455,\tT: 2,793,258,\tG: 15,\tavg_r: 0.00759,\tavg_G: 17.8,\tt: 1976,\tdt: 2.805ms,\tSimpleTD/loss: 0.00516,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00243\n",
      "INFO:TrainMonitor:ep: 1456,\tT: 2,795,420,\tG: 12,\tavg_r: 0.00555,\tavg_G: 17.2,\tt: 2161,\tdt: 2.767ms,\tSimpleTD/loss: 0.00603,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.0024\n",
      "INFO:TrainMonitor:ep: 1457,\tT: 2,797,141,\tG: 19,\tavg_r: 0.011,\tavg_G: 17.4,\tt: 1720,\tdt: 2.795ms,\tSimpleTD/loss: 0.0039,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00214\n",
      "INFO:TrainMonitor:ep: 1458,\tT: 2,799,146,\tG: 14,\tavg_r: 0.00699,\tavg_G: 17.1,\tt: 2004,\tdt: 2.802ms,\tSimpleTD/loss: 0.00557,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00359\n",
      "INFO:TrainMonitor:ep: 1459,\tT: 2,801,118,\tG: 16,\tavg_r: 0.00812,\tavg_G: 17,\tt: 1971,\tdt: 5.520ms,\tSimpleTD/loss: 0.00387,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00395\n",
      "INFO:TrainMonitor:ep: 1460,\tT: 2,803,051,\tG: 15,\tavg_r: 0.00776,\tavg_G: 16.8,\tt: 1932,\tdt: 2.823ms,\tSimpleTD/loss: 0.00612,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00211\n",
      "INFO:TrainMonitor:ep: 1461,\tT: 2,804,905,\tG: 16,\tavg_r: 0.00863,\tavg_G: 16.7,\tt: 1853,\tdt: 2.790ms,\tSimpleTD/loss: 0.003,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00307\n",
      "INFO:TrainMonitor:ep: 1462,\tT: 2,807,186,\tG: 8,\tavg_r: 0.00351,\tavg_G: 15.8,\tt: 2280,\tdt: 2.805ms,\tSimpleTD/loss: 0.00748,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: -0.00318\n",
      "INFO:TrainMonitor:ep: 1463,\tT: 2,809,268,\tG: 13,\tavg_r: 0.00625,\tavg_G: 15.5,\tt: 2081,\tdt: 2.809ms,\tSimpleTD/loss: 0.00545,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.00325\n",
      "INFO:TrainMonitor:ep: 1464,\tT: 2,811,171,\tG: 16,\tavg_r: 0.00841,\tavg_G: 15.6,\tt: 1902,\tdt: 6.676ms,\tSimpleTD/loss: 0.00503,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00241\n",
      "INFO:TrainMonitor:ep: 1465,\tT: 2,813,038,\tG: 19,\tavg_r: 0.0102,\tavg_G: 15.9,\tt: 1866,\tdt: 2.790ms,\tSimpleTD/loss: 0.00371,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.0045\n",
      "INFO:TrainMonitor:ep: 1466,\tT: 2,814,987,\tG: 14,\tavg_r: 0.00719,\tavg_G: 15.7,\tt: 1948,\tdt: 2.778ms,\tSimpleTD/loss: 0.00591,\tPPOClip/EntropyRegularizer/entropy: 1.61,\tPPOClip/loss: -0.00281\n",
      "INFO:TrainMonitor:ep: 1467,\tT: 2,816,788,\tG: 17,\tavg_r: 0.00944,\tavg_G: 15.9,\tt: 1800,\tdt: 2.839ms,\tSimpleTD/loss: 0.0034,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00137\n",
      "INFO:TrainMonitor:ep: 1468,\tT: 2,818,771,\tG: 15,\tavg_r: 0.00757,\tavg_G: 15.8,\tt: 1982,\tdt: 2.778ms,\tSimpleTD/loss: 0.00727,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00287\n",
      "INFO:TrainMonitor:ep: 1469,\tT: 2,820,542,\tG: 18,\tavg_r: 0.0102,\tavg_G: 16,\tt: 1770,\tdt: 6.404ms,\tSimpleTD/loss: 0.00366,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00281\n",
      "INFO:TrainMonitor:ep: 1470,\tT: 2,822,288,\tG: 19,\tavg_r: 0.0109,\tavg_G: 16.3,\tt: 1745,\tdt: 2.816ms,\tSimpleTD/loss: 0.00342,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00272\n",
      "INFO:TrainMonitor:ep: 1471,\tT: 2,824,074,\tG: 17,\tavg_r: 0.00952,\tavg_G: 16.4,\tt: 1785,\tdt: 2.811ms,\tSimpleTD/loss: 0.0039,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00161\n",
      "INFO:TrainMonitor:ep: 1472,\tT: 2,825,955,\tG: 17,\tavg_r: 0.00904,\tavg_G: 16.4,\tt: 1880,\tdt: 2.790ms,\tSimpleTD/loss: 0.00446,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00276\n",
      "INFO:TrainMonitor:ep: 1473,\tT: 2,827,730,\tG: 19,\tavg_r: 0.0107,\tavg_G: 16.7,\tt: 1774,\tdt: 2.809ms,\tSimpleTD/loss: 0.00391,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.0047\n",
      "INFO:TrainMonitor:ep: 1474,\tT: 2,829,677,\tG: 15,\tavg_r: 0.00771,\tavg_G: 16.5,\tt: 1946,\tdt: 2.827ms,\tSimpleTD/loss: 0.00457,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00116\n",
      "INFO:TrainMonitor:ep: 1475,\tT: 2,831,334,\tG: 20,\tavg_r: 0.0121,\tavg_G: 16.9,\tt: 1656,\tdt: 5.944ms,\tSimpleTD/loss: 0.00106,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0032\n",
      "INFO:TrainMonitor:ep: 1476,\tT: 2,833,058,\tG: 19,\tavg_r: 0.011,\tavg_G: 17.1,\tt: 1723,\tdt: 2.805ms,\tSimpleTD/loss: 0.00183,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00228\n",
      "INFO:TrainMonitor:ep: 1477,\tT: 2,834,843,\tG: 19,\tavg_r: 0.0107,\tavg_G: 17.3,\tt: 1784,\tdt: 2.802ms,\tSimpleTD/loss: 0.00217,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00141\n",
      "INFO:TrainMonitor:ep: 1478,\tT: 2,836,564,\tG: 19,\tavg_r: 0.011,\tavg_G: 17.4,\tt: 1720,\tdt: 2.816ms,\tSimpleTD/loss: 0.00193,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00232\n",
      "INFO:TrainMonitor:ep: 1479,\tT: 2,838,223,\tG: 20,\tavg_r: 0.0121,\tavg_G: 17.7,\tt: 1658,\tdt: 2.774ms,\tSimpleTD/loss: 0.000866,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00233\n",
      "INFO:TrainMonitor:ep: 1480,\tT: 2,839,885,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.9,\tt: 1661,\tdt: 2.831ms,\tSimpleTD/loss: 0.000594,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00231\n",
      "INFO:TrainMonitor:ep: 1481,\tT: 2,841,636,\tG: 18,\tavg_r: 0.0103,\tavg_G: 17.9,\tt: 1750,\tdt: 5.972ms,\tSimpleTD/loss: 0.0025,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00321\n",
      "INFO:TrainMonitor:ep: 1482,\tT: 2,843,347,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18,\tt: 1710,\tdt: 2.773ms,\tSimpleTD/loss: 0.00253,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00215\n",
      "INFO:TrainMonitor:ep: 1483,\tT: 2,845,013,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.2,\tt: 1665,\tdt: 2.824ms,\tSimpleTD/loss: 0.000523,\tPPOClip/EntropyRegularizer/entropy: 1.63,\tPPOClip/loss: 0.000909\n",
      "INFO:TrainMonitor:ep: 1484,\tT: 2,846,714,\tG: 19,\tavg_r: 0.0112,\tavg_G: 18.3,\tt: 1700,\tdt: 2.772ms,\tSimpleTD/loss: 0.00162,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00242\n",
      "INFO:TrainMonitor:ep: 1485,\tT: 2,848,537,\tG: 18,\tavg_r: 0.00988,\tavg_G: 18.3,\tt: 1822,\tdt: 2.805ms,\tSimpleTD/loss: 0.00318,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.0032\n",
      "INFO:TrainMonitor:ep: 1486,\tT: 2,850,203,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.5,\tt: 1665,\tdt: 6.523ms,\tSimpleTD/loss: 0.00107,\tPPOClip/EntropyRegularizer/entropy: 1.62,\tPPOClip/loss: -0.0028\n",
      "INFO:TrainMonitor:ep: 1487,\tT: 2,851,866,\tG: 21,\tavg_r: 0.0126,\tavg_G: 18.7,\tt: 1662,\tdt: 2.758ms,\tSimpleTD/loss: 0.00295,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00117\n",
      "INFO:TrainMonitor:ep: 1488,\tT: 2,853,585,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.7,\tt: 1718,\tdt: 2.808ms,\tSimpleTD/loss: 0.00186,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00192\n",
      "INFO:TrainMonitor:ep: 1489,\tT: 2,855,636,\tG: 12,\tavg_r: 0.00585,\tavg_G: 18.1,\tt: 2050,\tdt: 2.796ms,\tSimpleTD/loss: 0.0105,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00401\n",
      "INFO:TrainMonitor:ep: 1490,\tT: 2,857,392,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18.2,\tt: 1755,\tdt: 2.798ms,\tSimpleTD/loss: 0.00294,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00216\n",
      "INFO:TrainMonitor:ep: 1491,\tT: 2,859,049,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.3,\tt: 1656,\tdt: 2.766ms,\tSimpleTD/loss: 0.000666,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00318\n",
      "INFO:TrainMonitor:ep: 1492,\tT: 2,860,741,\tG: 19,\tavg_r: 0.0112,\tavg_G: 18.4,\tt: 1691,\tdt: 6.127ms,\tSimpleTD/loss: 0.00152,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.000955\n",
      "INFO:TrainMonitor:ep: 1493,\tT: 2,862,495,\tG: 18,\tavg_r: 0.0103,\tavg_G: 18.4,\tt: 1753,\tdt: 2.799ms,\tSimpleTD/loss: 0.0024,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00222\n",
      "INFO:TrainMonitor:ep: 1494,\tT: 2,864,364,\tG: 18,\tavg_r: 0.00964,\tavg_G: 18.3,\tt: 1868,\tdt: 2.791ms,\tSimpleTD/loss: 0.00273,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00285\n",
      "INFO:TrainMonitor:ep: 1495,\tT: 2,866,058,\tG: 20,\tavg_r: 0.0118,\tavg_G: 18.5,\tt: 1693,\tdt: 2.815ms,\tSimpleTD/loss: 0.000781,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00253\n",
      "INFO:TrainMonitor:ep: 1496,\tT: 2,867,722,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.6,\tt: 1663,\tdt: 2.767ms,\tSimpleTD/loss: 0.000333,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00248\n",
      "INFO:TrainMonitor:ep: 1497,\tT: 2,869,432,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.7,\tt: 1709,\tdt: 2.820ms,\tSimpleTD/loss: 0.000283,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.0021\n",
      "INFO:TrainMonitor:ep: 1498,\tT: 2,871,096,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.8,\tt: 1663,\tdt: 6.576ms,\tSimpleTD/loss: 0.00228,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00235\n",
      "INFO:TrainMonitor:ep: 1499,\tT: 2,872,861,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18.8,\tt: 1764,\tdt: 2.759ms,\tSimpleTD/loss: 0.00399,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00355\n",
      "INFO:TrainMonitor:ep: 1500,\tT: 2,874,524,\tG: 20,\tavg_r: 0.012,\tavg_G: 19,\tt: 1662,\tdt: 2.810ms,\tSimpleTD/loss: 0.000602,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00203\n",
      "INFO:TrainMonitor:ep: 1501,\tT: 2,876,190,\tG: 20,\tavg_r: 0.012,\tavg_G: 19.1,\tt: 1665,\tdt: 2.804ms,\tSimpleTD/loss: 0.000416,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00234\n",
      "INFO:TrainMonitor:ep: 1502,\tT: 2,877,977,\tG: 18,\tavg_r: 0.0101,\tavg_G: 18.9,\tt: 1786,\tdt: 2.803ms,\tSimpleTD/loss: 0.00162,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00186\n",
      "INFO:TrainMonitor:ep: 1503,\tT: 2,880,017,\tG: 16,\tavg_r: 0.00785,\tavg_G: 18.7,\tt: 2039,\tdt: 5.837ms,\tSimpleTD/loss: 0.00412,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00322\n",
      "INFO:TrainMonitor:ep: 1504,\tT: 2,881,749,\tG: 19,\tavg_r: 0.011,\tavg_G: 18.7,\tt: 1731,\tdt: 2.810ms,\tSimpleTD/loss: 0.00376,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00379\n",
      "INFO:TrainMonitor:ep: 1505,\tT: 2,883,518,\tG: 18,\tavg_r: 0.0102,\tavg_G: 18.6,\tt: 1768,\tdt: 2.813ms,\tSimpleTD/loss: 0.00234,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.0025\n",
      "INFO:TrainMonitor:ep: 1506,\tT: 2,885,180,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.8,\tt: 1661,\tdt: 2.785ms,\tSimpleTD/loss: 0.000714,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00273\n",
      "INFO:TrainMonitor:ep: 1507,\tT: 2,886,960,\tG: 19,\tavg_r: 0.0107,\tavg_G: 18.8,\tt: 1779,\tdt: 2.798ms,\tSimpleTD/loss: 0.001,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0021\n",
      "INFO:TrainMonitor:ep: 1508,\tT: 2,888,739,\tG: 19,\tavg_r: 0.0107,\tavg_G: 18.8,\tt: 1778,\tdt: 2.811ms,\tSimpleTD/loss: 0.00139,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0032\n",
      "INFO:TrainMonitor:ep: 1509,\tT: 2,890,489,\tG: 20,\tavg_r: 0.0114,\tavg_G: 18.9,\tt: 1749,\tdt: 5.815ms,\tSimpleTD/loss: 0.00105,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00276\n",
      "INFO:TrainMonitor:ep: 1510,\tT: 2,892,251,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18.9,\tt: 1761,\tdt: 2.806ms,\tSimpleTD/loss: 0.00179,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00241\n",
      "INFO:TrainMonitor:ep: 1511,\tT: 2,894,162,\tG: 17,\tavg_r: 0.0089,\tavg_G: 18.7,\tt: 1910,\tdt: 2.828ms,\tSimpleTD/loss: 0.00264,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00249\n",
      "INFO:TrainMonitor:ep: 1512,\tT: 2,895,999,\tG: 16,\tavg_r: 0.00871,\tavg_G: 18.5,\tt: 1836,\tdt: 2.778ms,\tSimpleTD/loss: 0.00572,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00127\n",
      "INFO:TrainMonitor:ep: 1513,\tT: 2,897,661,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.6,\tt: 1661,\tdt: 2.763ms,\tSimpleTD/loss: 0.00109,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00397\n",
      "INFO:TrainMonitor:ep: 1514,\tT: 2,899,404,\tG: 19,\tavg_r: 0.0109,\tavg_G: 18.7,\tt: 1742,\tdt: 2.817ms,\tSimpleTD/loss: 0.00131,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00227\n",
      "INFO:TrainMonitor:ep: 1515,\tT: 2,901,149,\tG: 20,\tavg_r: 0.0115,\tavg_G: 18.8,\tt: 1744,\tdt: 5.956ms,\tSimpleTD/loss: 0.00159,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00355\n",
      "INFO:TrainMonitor:ep: 1516,\tT: 2,902,808,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.9,\tt: 1658,\tdt: 2.782ms,\tSimpleTD/loss: 0.000564,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00232\n",
      "INFO:TrainMonitor:ep: 1517,\tT: 2,904,561,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18.9,\tt: 1752,\tdt: 2.816ms,\tSimpleTD/loss: 0.00163,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00252\n",
      "INFO:TrainMonitor:ep: 1518,\tT: 2,906,320,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18.9,\tt: 1758,\tdt: 2.804ms,\tSimpleTD/loss: 0.0014,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00239\n",
      "INFO:TrainMonitor:ep: 1519,\tT: 2,908,165,\tG: 17,\tavg_r: 0.00922,\tavg_G: 18.7,\tt: 1844,\tdt: 2.815ms,\tSimpleTD/loss: 0.00315,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00177\n",
      "INFO:TrainMonitor:ep: 1520,\tT: 2,909,955,\tG: 19,\tavg_r: 0.0106,\tavg_G: 18.8,\tt: 1789,\tdt: 2.812ms,\tSimpleTD/loss: 0.00214,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00294\n",
      "INFO:TrainMonitor:ep: 1521,\tT: 2,911,671,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.8,\tt: 1715,\tdt: 6.177ms,\tSimpleTD/loss: 0.00169,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00264\n",
      "INFO:TrainMonitor:ep: 1522,\tT: 2,913,482,\tG: 18,\tavg_r: 0.00994,\tavg_G: 18.7,\tt: 1810,\tdt: 2.809ms,\tSimpleTD/loss: 0.00156,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00284\n",
      "INFO:TrainMonitor:ep: 1523,\tT: 2,915,221,\tG: 20,\tavg_r: 0.0115,\tavg_G: 18.8,\tt: 1738,\tdt: 2.825ms,\tSimpleTD/loss: 0.00219,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00261\n",
      "INFO:TrainMonitor:ep: 1524,\tT: 2,917,238,\tG: 15,\tavg_r: 0.00744,\tavg_G: 18.5,\tt: 2016,\tdt: 2.808ms,\tSimpleTD/loss: 0.00355,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00293\n",
      "INFO:TrainMonitor:ep: 1525,\tT: 2,919,054,\tG: 18,\tavg_r: 0.00992,\tavg_G: 18.4,\tt: 1815,\tdt: 2.795ms,\tSimpleTD/loss: 0.00271,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00342\n",
      "INFO:TrainMonitor:ep: 1526,\tT: 2,920,828,\tG: 19,\tavg_r: 0.0107,\tavg_G: 18.5,\tt: 1773,\tdt: 5.817ms,\tSimpleTD/loss: 0.00277,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00138\n",
      "INFO:TrainMonitor:ep: 1527,\tT: 2,922,584,\tG: 18,\tavg_r: 0.0103,\tavg_G: 18.4,\tt: 1755,\tdt: 2.816ms,\tSimpleTD/loss: 0.00303,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00258\n",
      "INFO:TrainMonitor:ep: 1528,\tT: 2,924,639,\tG: 16,\tavg_r: 0.00779,\tavg_G: 18.2,\tt: 2054,\tdt: 2.794ms,\tSimpleTD/loss: 0.00635,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00335\n",
      "INFO:TrainMonitor:ep: 1529,\tT: 2,926,522,\tG: 16,\tavg_r: 0.0085,\tavg_G: 18,\tt: 1882,\tdt: 2.783ms,\tSimpleTD/loss: 0.00459,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00297\n",
      "INFO:TrainMonitor:ep: 1530,\tT: 2,928,184,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.2,\tt: 1661,\tdt: 2.769ms,\tSimpleTD/loss: 0.00118,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00297\n",
      "INFO:TrainMonitor:ep: 1531,\tT: 2,929,898,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.2,\tt: 1713,\tdt: 2.808ms,\tSimpleTD/loss: 0.00153,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00251\n",
      "INFO:TrainMonitor:ep: 1532,\tT: 2,931,672,\tG: 19,\tavg_r: 0.0107,\tavg_G: 18.3,\tt: 1773,\tdt: 6.156ms,\tSimpleTD/loss: 0.0015,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00192\n",
      "INFO:TrainMonitor:ep: 1533,\tT: 2,933,908,\tG: 10,\tavg_r: 0.00447,\tavg_G: 17.5,\tt: 2235,\tdt: 2.838ms,\tSimpleTD/loss: 0.00751,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00402\n",
      "INFO:TrainMonitor:ep: 1534,\tT: 2,935,620,\tG: 19,\tavg_r: 0.0111,\tavg_G: 17.6,\tt: 1711,\tdt: 2.762ms,\tSimpleTD/loss: 0.00238,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00204\n",
      "INFO:TrainMonitor:ep: 1535,\tT: 2,937,595,\tG: 15,\tavg_r: 0.0076,\tavg_G: 17.4,\tt: 1974,\tdt: 2.835ms,\tSimpleTD/loss: 0.00637,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00347\n",
      "INFO:TrainMonitor:ep: 1536,\tT: 2,939,258,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.6,\tt: 1662,\tdt: 2.837ms,\tSimpleTD/loss: 0.0013,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00323\n",
      "INFO:TrainMonitor:ep: 1537,\tT: 2,940,923,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.9,\tt: 1664,\tdt: 6.012ms,\tSimpleTD/loss: 0.000583,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.0023\n",
      "INFO:TrainMonitor:ep: 1538,\tT: 2,942,972,\tG: 14,\tavg_r: 0.00684,\tavg_G: 17.5,\tt: 2048,\tdt: 2.821ms,\tSimpleTD/loss: 0.00523,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00369\n",
      "INFO:TrainMonitor:ep: 1539,\tT: 2,944,667,\tG: 20,\tavg_r: 0.0118,\tavg_G: 17.7,\tt: 1694,\tdt: 2.818ms,\tSimpleTD/loss: 0.00284,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00274\n",
      "INFO:TrainMonitor:ep: 1540,\tT: 2,946,469,\tG: 17,\tavg_r: 0.00944,\tavg_G: 17.7,\tt: 1801,\tdt: 2.803ms,\tSimpleTD/loss: 0.00427,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00362\n",
      "INFO:TrainMonitor:ep: 1541,\tT: 2,948,260,\tG: 18,\tavg_r: 0.0101,\tavg_G: 17.7,\tt: 1790,\tdt: 2.816ms,\tSimpleTD/loss: 0.00437,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00255\n",
      "INFO:TrainMonitor:ep: 1542,\tT: 2,949,922,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.9,\tt: 1661,\tdt: 2.787ms,\tSimpleTD/loss: 0.00063,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0024\n",
      "INFO:TrainMonitor:ep: 1543,\tT: 2,951,840,\tG: 18,\tavg_r: 0.00939,\tavg_G: 17.9,\tt: 1917,\tdt: 5.570ms,\tSimpleTD/loss: 0.00444,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00254\n",
      "INFO:TrainMonitor:ep: 1544,\tT: 2,953,471,\tG: 21,\tavg_r: 0.0129,\tavg_G: 18.2,\tt: 1630,\tdt: 2.776ms,\tSimpleTD/loss: 0.00135,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00257\n",
      "INFO:TrainMonitor:ep: 1545,\tT: 2,955,538,\tG: 16,\tavg_r: 0.00774,\tavg_G: 18,\tt: 2066,\tdt: 2.789ms,\tSimpleTD/loss: 0.00443,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00416\n",
      "INFO:TrainMonitor:ep: 1546,\tT: 2,957,205,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.2,\tt: 1666,\tdt: 2.823ms,\tSimpleTD/loss: 0.000747,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00265\n",
      "INFO:TrainMonitor:ep: 1547,\tT: 2,958,920,\tG: 19,\tavg_r: 0.0111,\tavg_G: 18.3,\tt: 1714,\tdt: 2.766ms,\tSimpleTD/loss: 0.00132,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00178\n",
      "INFO:TrainMonitor:ep: 1548,\tT: 2,960,586,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.5,\tt: 1665,\tdt: 5.971ms,\tSimpleTD/loss: 0.000597,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00284\n",
      "INFO:TrainMonitor:ep: 1549,\tT: 2,962,483,\tG: 15,\tavg_r: 0.00791,\tavg_G: 18.1,\tt: 1896,\tdt: 2.779ms,\tSimpleTD/loss: 0.00365,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00131\n",
      "INFO:TrainMonitor:ep: 1550,\tT: 2,964,195,\tG: 20,\tavg_r: 0.0117,\tavg_G: 18.3,\tt: 1711,\tdt: 2.814ms,\tSimpleTD/loss: 0.00188,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00324\n",
      "INFO:TrainMonitor:ep: 1551,\tT: 2,965,919,\tG: 19,\tavg_r: 0.011,\tavg_G: 18.4,\tt: 1723,\tdt: 2.813ms,\tSimpleTD/loss: 0.00198,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00175\n",
      "INFO:TrainMonitor:ep: 1552,\tT: 2,967,580,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.5,\tt: 1660,\tdt: 2.788ms,\tSimpleTD/loss: 0.000523,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00267\n",
      "INFO:TrainMonitor:ep: 1553,\tT: 2,969,460,\tG: 17,\tavg_r: 0.00905,\tavg_G: 18.4,\tt: 1879,\tdt: 2.842ms,\tSimpleTD/loss: 0.00342,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00244\n",
      "INFO:TrainMonitor:ep: 1554,\tT: 2,971,123,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.5,\tt: 1662,\tdt: 6.869ms,\tSimpleTD/loss: 0.000571,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00268\n",
      "INFO:TrainMonitor:ep: 1555,\tT: 2,972,784,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.7,\tt: 1660,\tdt: 2.786ms,\tSimpleTD/loss: 0.000614,\tPPOClip/EntropyRegularizer/entropy: 1.59,\tPPOClip/loss: -0.00252\n",
      "INFO:TrainMonitor:ep: 1556,\tT: 2,974,664,\tG: 17,\tavg_r: 0.00905,\tavg_G: 18.5,\tt: 1879,\tdt: 2.837ms,\tSimpleTD/loss: 0.00291,\tPPOClip/EntropyRegularizer/entropy: 1.6,\tPPOClip/loss: -0.00253\n",
      "INFO:TrainMonitor:ep: 1557,\tT: 2,976,619,\tG: 16,\tavg_r: 0.00819,\tavg_G: 18.3,\tt: 1954,\tdt: 2.776ms,\tSimpleTD/loss: 0.00594,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00398\n",
      "INFO:TrainMonitor:ep: 1558,\tT: 2,978,504,\tG: 17,\tavg_r: 0.00902,\tavg_G: 18.1,\tt: 1884,\tdt: 2.837ms,\tSimpleTD/loss: 0.00372,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00263\n",
      "INFO:TrainMonitor:ep: 1559,\tT: 2,980,661,\tG: 13,\tavg_r: 0.00603,\tavg_G: 17.6,\tt: 2156,\tdt: 5.876ms,\tSimpleTD/loss: 0.00543,\tPPOClip/EntropyRegularizer/entropy: 1.58,\tPPOClip/loss: -0.00334\n",
      "INFO:TrainMonitor:ep: 1560,\tT: 2,982,414,\tG: 18,\tavg_r: 0.0103,\tavg_G: 17.7,\tt: 1752,\tdt: 2.811ms,\tSimpleTD/loss: 0.00364,\tPPOClip/EntropyRegularizer/entropy: 1.55,\tPPOClip/loss: -0.00145\n",
      "INFO:TrainMonitor:ep: 1561,\tT: 2,984,255,\tG: 18,\tavg_r: 0.00978,\tavg_G: 17.7,\tt: 1840,\tdt: 2.806ms,\tSimpleTD/loss: 0.00199,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00203\n",
      "INFO:TrainMonitor:ep: 1562,\tT: 2,986,093,\tG: 18,\tavg_r: 0.0098,\tavg_G: 17.7,\tt: 1837,\tdt: 2.791ms,\tSimpleTD/loss: 0.00562,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00438\n",
      "INFO:TrainMonitor:ep: 1563,\tT: 2,987,877,\tG: 17,\tavg_r: 0.00953,\tavg_G: 17.7,\tt: 1783,\tdt: 2.794ms,\tSimpleTD/loss: 0.00281,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00184\n",
      "INFO:TrainMonitor:ep: 1564,\tT: 2,989,570,\tG: 19,\tavg_r: 0.0112,\tavg_G: 17.8,\tt: 1692,\tdt: 2.827ms,\tSimpleTD/loss: 0.00134,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00182\n",
      "INFO:TrainMonitor:ep: 1565,\tT: 2,991,461,\tG: 16,\tavg_r: 0.00847,\tavg_G: 17.6,\tt: 1890,\tdt: 6.068ms,\tSimpleTD/loss: 0.00362,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00223\n",
      "INFO:TrainMonitor:ep: 1566,\tT: 2,993,123,\tG: 20,\tavg_r: 0.012,\tavg_G: 17.9,\tt: 1661,\tdt: 2.846ms,\tSimpleTD/loss: 0.000958,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00296\n",
      "INFO:TrainMonitor:ep: 1567,\tT: 2,994,782,\tG: 20,\tavg_r: 0.0121,\tavg_G: 18.1,\tt: 1658,\tdt: 2.787ms,\tSimpleTD/loss: 0.000799,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.0021\n",
      "INFO:TrainMonitor:ep: 1568,\tT: 2,996,445,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.3,\tt: 1662,\tdt: 2.824ms,\tSimpleTD/loss: 0.000796,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00255\n",
      "INFO:TrainMonitor:ep: 1569,\tT: 2,998,200,\tG: 19,\tavg_r: 0.0108,\tavg_G: 18.3,\tt: 1754,\tdt: 2.820ms,\tSimpleTD/loss: 0.00133,\tPPOClip/EntropyRegularizer/entropy: 1.57,\tPPOClip/loss: -0.00212\n",
      "INFO:TrainMonitor:ep: 1570,\tT: 2,999,861,\tG: 20,\tavg_r: 0.012,\tavg_G: 18.5,\tt: 1660,\tdt: 2.781ms,\tSimpleTD/loss: 0.000349,\tPPOClip/EntropyRegularizer/entropy: 1.56,\tPPOClip/loss: -0.00225\n"
     ]
    }
   ],
   "source": [
    "while environment.T < 3000000:\n",
    "    s, info = environment.reset()\n",
    "\n",
    "    for t in range(environment.spec.max_episode_steps):\n",
    "        a, logp = pi_behavior(s, return_logp=True)\n",
    "        s_next, r, done, truncated, info = environment.step(a)\n",
    "\n",
    "        tracer.add(s, a, r, done, logp)\n",
    "        while tracer:\n",
    "            buffer.add(tracer.pop())\n",
    "\n",
    "        if len(buffer) >= buffer.capacity:\n",
    "            num_batches = int(4 * buffer.capacity / 32)\n",
    "            for _ in range(num_batches):\n",
    "                transition_batch = buffer.sample(32)\n",
    "                metrics_v, td_error = simpletd.update(transition_batch, return_td_error=True)\n",
    "                metrics_pi = ppo_clip.update(transition_batch, td_error)\n",
    "                environment.record_metrics(metrics_v)\n",
    "                environment.record_metrics(metrics_pi)\n",
    "\n",
    "            buffer.clear()\n",
    "\n",
    "            pi_behavior.soft_update(pi, tau=0.1)\n",
    "            v_targ.soft_update(v, tau=0.1)\n",
    "\n",
    "        if done or truncated:\n",
    "            break\n",
    "\n",
    "        s = s_next\n",
    "\n",
    "    if environment.period(name='generate_gif', T_period=10000):\n",
    "        T = environment.T - environment.T % 10000  # round to 10000s\n",
    "        coax.utils.generate_gif(\n",
    "            env=environment, policy=pi, resize_to=(320, 420),\n",
    "            filepath=f\"./data/gifs/{name}/T{T:08d}.gif\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3cc509e8-1bc2-43f1-8927-064e698cf414",
   "metadata": {},
   "source": [
    "# Save Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "24f23062-e018-4f59-8b11-ce83b306d27b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-28T02:19:31.421993Z",
     "iopub.status.busy": "2023-11-28T02:19:31.421910Z",
     "iopub.status.idle": "2023-11-28T02:19:31.455380Z",
     "shell.execute_reply": "2023-11-28T02:19:31.455029Z",
     "shell.execute_reply.started": "2023-11-28T02:19:31.421985Z"
    }
   },
   "outputs": [],
   "source": [
    "coax.utils.dump((pi, v, pi_behavior, v_targ), 'checkpoint.pkl.lz4')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}