├── test ├── test_helper.exs └── gym_tcp_api_test.exs ├── lib ├── node_manager.ex ├── gym_tcp_api.ex ├── worker.ex └── server.ex ├── mix.lock ├── .gitignore ├── cpp ├── record_episode_statistics_impl.hpp ├── space_impl.hpp ├── README.md ├── record_episode_statistics.hpp ├── example.cpp ├── space.hpp ├── CMakeLists.txt ├── parser.hpp ├── environment_impl.hpp ├── environment.hpp ├── messages.hpp ├── parser_impl.hpp ├── client.hpp └── pjson │ └── pjson.h ├── mix.exs ├── config └── config.exs ├── LICENSE ├── README.md ├── priv └── worker.py └── python └── server.py /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | -------------------------------------------------------------------------------- /test/gym_tcp_api_test.exs: -------------------------------------------------------------------------------- 1 | defmodule GymTcpApiTest do 2 | use ExUnit.Case 3 | doctest GymTcpApi 4 | 5 | test "the truth" do 6 | assert 1 + 1 == 2 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /lib/node_manager.ex: -------------------------------------------------------------------------------- 1 | _ = """ 2 | @file node_manager.ex 3 | @author Marcus Edel 4 | 5 | Node handler. 6 | """ 7 | 8 | defmodule GymTcpApi.NodeManager do 9 | def all_nodes do 10 | Node.list ++ [Node.self] 11 | end 12 | 13 | def random_node(_) do 14 | Enum.random(all_nodes()) 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "erlport": {:hex, :erlport, "0.10.0", "2436ec2f4ed62538c6e9c52f523f9315b6002ee7e298d9bd10b35abc3f6b32e7", [:rebar3], [], "hexpm", "bdb61bc3a50c1e06945eab16f6f459160169e9daefcef5072948c13509a5ee92"}, 3 | "poolboy": {:hex, :poolboy, "1.5.1", "6b46163901cfd0a1b43d692657ed9d7e599853b3b21b95ae5ae0a777cf9b6ca8", [:rebar], [], "hexpm", "8f7168911120e13419e086e78d20e4d1a6776f1eee2411ac9f790af10813389f"}, 4 | } 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps 9 | 10 | # Where 3rd-party dependencies like ExDoc output generated docs. 11 | /doc 12 | 13 | # If the VM crashes, it generates a dump, let's ignore it too. 14 | erl_crash.dump 15 | 16 | # Also ignore archive artifacts (built via "mix archive.build"). 17 | *.ez 18 | 19 | # The c++ build directory. 20 | build* 21 | 22 | # Python byte code. 23 | *.pyc 24 | 25 | # Dummy folder with all gym recordings and metadata 26 | /dummy 27 | -------------------------------------------------------------------------------- /cpp/record_episode_statistics_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file record_episode_statistics_impl.hpp 3 | * @author Nabanita Dash 4 | * 5 | * Implementation of miscellaneous record episode statistics routines. 6 | */ 7 | #ifndef GYM_RECORD_EPISODE_STATISTICS_IMPL_HPP 8 | #define GYM_RECORD_EPISODE_STATISTICS_IMPL_HPP 9 | 10 | // In case it hasn't been included yet. 11 | #include "record_episode_statistics.hpp" 12 | #include "messages.hpp" 13 | 14 | namespace gym { 15 | 16 | inline RecordEpisodeStatistics::RecordEpisodeStatistics() 17 | { 18 | // Nothing to do here. 19 | } 20 | 21 | inline void RecordEpisodeStatistics::client(Client& c) 22 | { 23 | clientPtr = &c; 24 | } 25 | 26 | inline void RecordEpisodeStatistics::start() 27 | { 28 | clientPtr->send(messages::RecordEpisodeStatisticsStart()); 29 | } 30 | 31 | } // namespace gym 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule GymTcpApi.Mixfile do 2 | use Mix.Project 3 | 4 | def project do 5 | [app: :gym_tcp_api, 6 | version: "0.1.0", 7 | elixir: "~> 1.0", 8 | build_embedded: Mix.env == :prod, 9 | start_permanent: Mix.env == :prod, 10 | deps: deps()] 11 | end 12 | 13 | # Configuration for the OTP application 14 | # 15 | # Type "mix help compile.app" for more information 16 | def application do 17 | [ 18 | mod: {GymTcpApi, []}, 19 | applications: [:logger, :poolboy] 20 | ] 21 | end 22 | 23 | # Dependencies can be Hex packages: 24 | # 25 | # {:mydep, "~> 0.3.0"} 26 | # 27 | # Or git/path repositories: 28 | # 29 | # {:mydep, git: "https://github.com/elixir-lang/mydep.git", tag: "0.1.0"} 30 | # 31 | # Type "mix help deps" for more examples and options 32 | defp deps do 33 | [ 34 | {:erlport, "~> 0.10.0"}, 35 | {:poolboy, "~> 1.5"} 36 | ] 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /cpp/space_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file space_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of miscellaneous space routines. 6 | */ 7 | #ifndef GYM_SPACE_IMPL_HPP 8 | #define GYM_SPACE_IMPL_HPP 9 | 10 | // In case it hasn't been included yet. 11 | #include "space.hpp" 12 | #include "parser.hpp" 13 | #include "messages.hpp" 14 | 15 | namespace gym { 16 | 17 | inline Space::Space() : parser(new Parser()) 18 | { 19 | // Nothing to do here. 20 | } 21 | 22 | inline Space::~Space() 23 | { 24 | delete parser; 25 | } 26 | 27 | inline void Space::client(Client& c) 28 | { 29 | clientPtr = &c; 30 | } 31 | 32 | inline const arma::mat& Space::sample() 33 | { 34 | clientPtr->send(messages::EnvironmentActionSpaceSample()); 35 | 36 | std::string json; 37 | clientPtr->receive(json); 38 | 39 | parser->parse(json); 40 | parser->actionSample(this, actionSample); 41 | 42 | return actionSample; 43 | } 44 | 45 | } // namespace gym 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /cpp/README.md: -------------------------------------------------------------------------------- 1 | # c++ client 2 | 3 | ## Dependencies 4 | 5 | The c++ example agent has the following dependencies: 6 | 7 | Armadillo >= 4.200.0 8 | Boost (system, thread) 9 | CMake >= 2.8.5 10 | 11 | ## Installation 12 | 13 | First, checkout the repository and change into the unpacked c++ client directory: 14 | 15 | git clone https://github.com/zoq/gym_tcp_api.git 16 | cd gym_tcp_api/cpp/ 17 | 18 | Then, make a build directory. The directory can have any name, not just 'build', but 'build' is sufficient. 19 | 20 | $ mkdir build 21 | $ cd build 22 | 23 | The next step is to run CMake to configure the project. Running CMake is the equivalent to running ./configure with autotools 24 | 25 | $ cmake ../ 26 | 27 | Once CMake is configured, building the the example agent is as simple as typing 'make'. 28 | 29 | $ make 30 | 31 | In a separate terminal, you can then run the example agent: 32 | 33 | $ ./example 34 | -------------------------------------------------------------------------------- /lib/gym_tcp_api.ex: -------------------------------------------------------------------------------- 1 | _ = """ 2 | @file gym_tcp_api.ex 3 | @author Marcus Edel 4 | 5 | GymTcpApi handler. 6 | """ 7 | 8 | defmodule GymTcpApi do 9 | use Application 10 | 11 | def start(_type, _args) do 12 | import Supervisor.Spec 13 | 14 | poolboy_config = [ 15 | {:name, {:local, pool_name()}}, 16 | {:worker_module, GymTcpApi.Worker}, 17 | {:size, Application.get_env(:gym_tcp_api, :worker)}, 18 | {:max_overflow, 0}, 19 | {:strategy, :fifo} 20 | ] 21 | 22 | children = [ 23 | :poolboy.child_spec(pool_name(), poolboy_config, []), 24 | supervisor(Task.Supervisor, [[name: GymTcpApi.TaskSupervisor]]), 25 | worker(Task, [GymTcpApi.Server, :accept, 26 | [Application.get_env(:gym_tcp_api, :port)]]) 27 | ] 28 | 29 | options = [ 30 | strategy: :one_for_one, 31 | name: GymTcpApi.Supervisor 32 | ] 33 | 34 | Supervisor.start_link(children, options) 35 | end 36 | 37 | def pool_name() do 38 | :gym_pool 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /cpp/record_episode_statistics.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file record_episode_statistics.hpp 3 | * @author Nabanita Dash 4 | * 5 | * Definition of miscellaneous record episode statistics routines. 6 | */ 7 | #ifndef GYM_RECORD_EPISODE_STATISTICS_HPP 8 | #define GYM_RECORD_EPISODE_STATISTICS_HPP 9 | 10 | #include "client.hpp" 11 | 12 | namespace gym { 13 | 14 | 15 | /* 16 | * Definition of the class. 17 | */ 18 | class RecordEpisodeStatistics 19 | { 20 | public: 21 | /** 22 | * Create the Parser object. 23 | */ 24 | RecordEpisodeStatistics(); 25 | 26 | /** 27 | * Set the client which is connected with the server. 28 | * 29 | * @param c The client object which is connected with the server. 30 | */ 31 | void client(Client& c); 32 | 33 | /* 34 | * Start the using the specified parameter. 35 | */ 36 | void start(); 37 | 38 | private: 39 | //! Locally-stored client pointer used to communicate with the connected 40 | //! server. 41 | Client* clientPtr; 42 | }; 43 | } // namespace gym 44 | 45 | // Include implementation. 46 | #include "record_episode_statistics_impl.hpp" 47 | 48 | #endif -------------------------------------------------------------------------------- /lib/worker.ex: -------------------------------------------------------------------------------- 1 | _ = """ 2 | @file worker.ex 3 | @author Marcus Edel 4 | 5 | Worker module. 6 | """ 7 | 8 | defmodule GymTcpApi.Worker do 9 | use GenServer 10 | 11 | require Logger 12 | 13 | alias Application, as: App 14 | 15 | def start_link(_args) do 16 | priv_path = App.app_dir(:gym_tcp_api, "priv") |> to_charlist 17 | GenServer.start_link(__MODULE__, priv_path) 18 | end 19 | 20 | def init(python_path) do 21 | :python.start_link(python_path: python_path) 22 | end 23 | 24 | def handle_call({data, caller}, _, python) do 25 | response = :python.call(python, :worker, :process_response, [data]) 26 | 27 | current = self() 28 | send(caller, {:response, response, current}) 29 | 30 | receive do 31 | {:data, message, _c } -> 32 | handle_call({message, caller}, :ok, python); 33 | {:close, _message} -> :python.call(python, :worker, :process_response, [""]) 34 | end 35 | 36 | {:reply, [""], python} 37 | end 38 | 39 | def process(pid, data, caller) do 40 | :gen_server.call(pid, {data, caller}); 41 | :poolboy.checkin(GymTcpApi.pool_name(), pid) 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your application 2 | # and its dependencies with the aid of the Mix.Config module. 3 | use Mix.Config 4 | 5 | # This configuration is loaded before any dependency and is restricted 6 | # to this project. If another project depends on this project, this 7 | # file won't be loaded nor affect the parent project. For this reason, 8 | # if you want to provide default values for your application for 9 | # 3rd-party users, it should be done in your "mix.exs" file. 10 | 11 | # You can configure for your application as: 12 | # 13 | # config :gym_tcp_api, key: :value 14 | # 15 | # And access this configuration in your application as: 16 | # 17 | # Application.get_env(:gym_tcp_api, :key) 18 | # 19 | # Or configure a 3rd-party app: 20 | # 21 | # config :logger, level: :info 22 | # 23 | 24 | # It is also possible to import configuration files, relative to this 25 | # directory. For example, you can emulate configuration per environment 26 | # by uncommenting the line below and defining dev.exs, test.exs and such. 27 | # Configuration from the imported file will override the ones defined 28 | # here (which is why it is important to import them last). 29 | # 30 | # import_config "#{Mix.env}.exs" 31 | 32 | config :gym_tcp_api, 33 | port: 4040, 34 | worker: 2, 35 | distributed: false 36 | -------------------------------------------------------------------------------- /cpp/example.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file example.cpp 3 | * @author Marcus Edel 4 | * 5 | * Simple random agent. 6 | */ 7 | 8 | #include 9 | 10 | #include "environment.hpp" 11 | 12 | using namespace gym; 13 | 14 | int main(int argc, char* argv[]) 15 | { 16 | const std::string environment = "CartPole-v1"; 17 | const std::string host = "127.0.0.1"; 18 | const std::string port = "4040"; 19 | 20 | double totalReward = 0; 21 | size_t totalSteps = 0; 22 | 23 | Environment env(host, port, environment); 24 | env.compression(9); 25 | env.record_episode_stats.start(); 26 | 27 | env.reset(); 28 | env.render(); 29 | 30 | while (1) 31 | { 32 | arma::mat action = env.action_space.sample(); 33 | std::cout << "action: \n" << action << std::endl; 34 | 35 | env.step(action); 36 | 37 | totalReward += env.reward; 38 | totalSteps += 1; 39 | 40 | if (env.done) 41 | { 42 | break; 43 | } 44 | 45 | std::cout << "Current step: " << totalSteps << " current reward: " 46 | << totalReward << std::endl; 47 | } 48 | 49 | std::cout << "Instance: " << env.instance << " total steps: " << totalSteps 50 | << " reward: " << totalReward << std::endl; 51 | 52 | env.close(); 53 | const std::string url = env.url(); 54 | 55 | std::cout << "Video: https://kurg.org/media/gym/" << url 56 | << " (it might take some minutes before the video is accessible)." 57 | << std::endl; 58 | 59 | return 0; 60 | } 61 | -------------------------------------------------------------------------------- /cpp/space.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file space.hpp 3 | * @author Marcus Edel 4 | * 5 | * Defenition of miscellaneous space routines. 6 | */ 7 | 8 | #ifndef GYM_SPACE_HPP 9 | #define GYM_SPACE_HPP 10 | 11 | #include 12 | #include 13 | 14 | #include "client.hpp" 15 | 16 | namespace gym { 17 | 18 | class Parser; 19 | 20 | /** 21 | * Definition of the space class. 22 | */ 23 | class Space 24 | { 25 | public: 26 | /** 27 | * Instantiate the Environment object. 28 | */ 29 | Space(); 30 | 31 | /** 32 | * Destroy the space object. 33 | */ 34 | ~Space(); 35 | 36 | /** 37 | * Set the client which is connected with the server. 38 | * 39 | * @param c The client object which is connected with the server. 40 | */ 41 | void client(Client& c); 42 | 43 | /* 44 | * Return a sample action. 45 | */ 46 | const arma::mat& sample(); 47 | 48 | //! Space type defenition. 49 | enum SpaceType 50 | { 51 | DISCRETE, 52 | BOX, 53 | MULTIDISCRETE, 54 | } type; 55 | 56 | //! Observation and action space information. 57 | std::vector boxShape; 58 | std::vector boxHigh; 59 | std::vector boxLow; 60 | int n; 61 | 62 | private: 63 | //! Locally-stored parser pointer. 64 | Parser* parser; 65 | 66 | //! Locally-stored client pointer. 67 | Client* clientPtr; 68 | 69 | //! Locally-stored action sample. 70 | arma::mat actionSample; 71 | }; 72 | 73 | } // namespace gym 74 | 75 | // Include implementation. 76 | #include "space_impl.hpp" 77 | 78 | #endif -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2016, Marcus Edel 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /lib/server.ex: -------------------------------------------------------------------------------- 1 | _ = """ 2 | @file server.ex 3 | @author Marcus Edel 4 | 5 | Server module. 6 | """ 7 | 8 | defmodule GymTcpApi.Server do 9 | require Logger 10 | 11 | alias GymTcpApi.NodeManager 12 | 13 | def accept(port) do 14 | opt = [:binary, packet: :line, active: false, reuseaddr: true] 15 | 16 | case :gen_tcp.listen(port, opt) do 17 | {:ok, socket} = {:ok, socket} -> 18 | Logger.info "Accepting connections on port #{port}"; 19 | loop_acceptor(socket); 20 | {:error, :eaddrinuse} = _ -> 21 | Logger.info "Already accepting connections on port #{port}."; 22 | end 23 | end 24 | 25 | defp loop_acceptor(socket) do 26 | {:ok, client} = :gen_tcp.accept(socket) 27 | 28 | {:ok, pid} = Task.Supervisor.start_child(GymTcpApi.TaskSupervisor, 29 | fn -> serve(client) end) 30 | :ok = :gen_tcp.controlling_process(client, pid) 31 | loop_acceptor(socket) 32 | end 33 | 34 | defp serve(socket) do 35 | case :gen_tcp.recv(socket, 0, 5000) do 36 | {:ok, data} = _ -> 37 | node = NodeManager.random_node(data) 38 | current = self() 39 | 40 | if Application.get_env(:gym_tcp_api, :distributed) == true do 41 | Node.spawn(node, __MODULE__, :pool_process, [data, current]) 42 | else 43 | pool_process(data, current) 44 | end 45 | 46 | receive do 47 | {:response, response, worker} -> 48 | write_line(socket, {:ok, response}); 49 | handle(socket, worker) 50 | end 51 | {:error, :timeout} = _ -> 52 | exit(:shutdown); 53 | {:error, :closed} = _ -> 54 | exit(:shutdown); 55 | {:error, _} = error -> 56 | exit(error); 57 | end 58 | end 59 | 60 | defp handle(socket, worker) do 61 | case :gen_tcp.recv(socket, 0, 5000) do 62 | {:ok, data} = _ -> 63 | current = self() 64 | send(worker, {:data, data, current}) 65 | 66 | receive do 67 | {:response, response, worker} -> 68 | write_line(socket, {:ok, response}) 69 | handle(socket, worker) 70 | end 71 | 72 | {:error, :timeout} = _ -> 73 | send(worker, {:close, "timeout"}) 74 | exit(:shutdown); 75 | {:error, :closed} = _ -> 76 | send(worker, {:close, "shutdown"}) 77 | exit(:shutdown); 78 | {:error, _} = error -> 79 | send(worker, {:close, "error"}) 80 | exit(error); 81 | end 82 | end 83 | 84 | def read_line(socket) do 85 | :gen_tcp.recv(socket, 0) 86 | end 87 | 88 | def write_line(socket, {:ok, text}) do 89 | if String.trim(to_string(text)) === "" do 90 | else 91 | :gen_tcp.send(socket, text) 92 | end 93 | end 94 | 95 | def pool_process(data, caller) do 96 | worker = :poolboy.checkout(:gym_pool) 97 | spawn(fn() -> GymTcpApi.Worker.process(worker, data, caller) end) 98 | end 99 | end 100 | -------------------------------------------------------------------------------- /cpp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 2.8) 2 | project(gym-tcp-api) 3 | 4 | # Include modules in the CMake directory. 5 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/CMake") 6 | 7 | # Set the CFLAGS and CXXFLAGS. 8 | add_definitions("-Wno-deprecated-declarations") 9 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/modules/") 10 | SET(CMAKE_CXX_FLAGS "-std=c++11 -O3 -fPIC -ffast-math ${CMAKE_CXX_FLAGS}") 11 | SET(CMAKE_C_FLAGS "-std=c99 -O3 ${CMAKE_C_FLAGS} -lz") 12 | 13 | # If using clang, we have to link against libc++ depending on the 14 | # OS (at least on some systems). Further, gcc sometimes optimizes calls to 15 | # math.h functions, making -lm unnecessary with gcc, but it may still be 16 | # necessary with clang. 17 | if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") 18 | IF (APPLE) 19 | # detect OS X version. Use '/usr/bin/sw_vers -productVersion' to 20 | # extract V from '10.V.x'.) 21 | EXEC_PROGRAM(/usr/bin/sw_vers ARGS 22 | -productVersion OUTPUT_VARIABLE MACOSX_VERSION_RAW) 23 | string(REGEX REPLACE 24 | "10\\.([0-9]+).*" "\\1" 25 | MACOSX_VERSION 26 | "${MACOSX_VERSION_RAW}") 27 | 28 | # OSX Lion (10.7) and OS X Mountain Lion (10.8) doesn't automatically 29 | # select the right stdlib. 30 | if(${MACOSX_VERSION} LESS 9) 31 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") 32 | endif(${MACOSX_VERSION} LESS 9) 33 | endif(APPLE) 34 | endif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") 35 | 36 | # Find the libraries we need to compile against. 37 | set(Boost_ADDITIONAL_VERSIONS 38 | "1.72.0" "1.72" 39 | "1.71.0" "1.71" 40 | "1.70.0" "1.70" 41 | "1.69.0" "1.69" 42 | "1.68.0" "1.68" 43 | "1.67.0" "1.67" 44 | "1.66.0" "1.66" 45 | "1.65.1" "1.65.0" "1.65" 46 | "1.64.1" "1.64.0" "1.64" 47 | "1.63.1" "1.63.0" "1.63" 48 | "1.62.1" "1.62.0" "1.62" 49 | "1.61.1" "1.61.0" "1.61" 50 | "1.60.1" "1.60.0" "1.60" 51 | "1.59.1" "1.59.0" "1.59" 52 | "1.58.1" "1.58.0" "1.58") 53 | 54 | find_package(Boost 1.49 COMPONENTS system thread iostreams) 55 | 56 | if (NOT Boost_FOUND) 57 | # Try again with non-multithreaded libraries ('mt' tag). 58 | set(Boost_USE_MULTITHREADED OFF) 59 | find_package(Boost 1.49 COMPONENTS system thread iostreams REQUIRED) 60 | endif() 61 | 62 | find_package(Armadillo 3.6.0 REQUIRED) 63 | 64 | # Include directories for the dependencies. 65 | include_directories(${CMAKE_SOURCE_DIR}/pjson) 66 | include_directories(${Boost_INCLUDE_DIRS}) 67 | 68 | include_directories(${CMAKE_CURRENT_BINARY_DIR}) 69 | include_directories(${CMAKE_SOURCE_DIR}) 70 | 71 | MESSAGE(STATUS ${Boost_LIBRARIES}) 72 | 73 | # Set source file path. 74 | set(gym_tcp_api_source 75 | example.cpp 76 | parser.hpp 77 | parser_impl.hpp 78 | space.hpp 79 | space_impl.hpp 80 | environment.hpp 81 | environment_impl.hpp 82 | client.hpp 83 | messages.hpp 84 | record_episode_statistics.hpp 85 | record_episode_statistics_impl.hpp 86 | ) 87 | 88 | # Define the executable and link against the libraries we need to build the 89 | # source. 90 | add_executable(example ${gym_tcp_api_source}) 91 | target_link_libraries(example ${Boost_LIBRARIES} ${ARMADILLO_LIBRARIES}) 92 | -------------------------------------------------------------------------------- /cpp/parser.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file parser.hpp 3 | * @author Marcus Edel 4 | * 5 | * Definition of miscellaneous parser routines. 6 | */ 7 | #ifndef GYM_PARSER_HPP 8 | #define GYM_PARSER_HPP 9 | 10 | #include 11 | #include 12 | 13 | #include "pjson/pjson.h" 14 | 15 | namespace gym { 16 | 17 | 18 | class Space; 19 | 20 | /** 21 | * Definition of a parser class that is used to parse the reponses. 22 | */ 23 | class Parser 24 | { 25 | public: 26 | /** 27 | * Create the Parser object. 28 | */ 29 | Parser(); 30 | 31 | /** 32 | * Create the Parser object using the specified json string and create the 33 | * tree to extract the attributes. 34 | * 35 | * @param data The data encoded as json string. 36 | */ 37 | Parser(const std::string& data); 38 | 39 | /* 40 | * Deconstructor to delete the datastream. 41 | */ 42 | ~Parser(); 43 | 44 | /** 45 | * Parse the specified json string and create a tree to extract the 46 | * attributes. 47 | * 48 | * @param data The data encoded as json string. 49 | */ 50 | void parse(const std::string& data); 51 | 52 | /** 53 | * Parse the observation data. 54 | * 55 | * @param space The space information class. 56 | * @param observation The parsed observation. 57 | */ 58 | void observation(const Space* space, arma::mat& observation); 59 | 60 | /** 61 | * Parse the space data. 62 | * 63 | * @param space The space information class. 64 | */ 65 | void space(Space* space); 66 | 67 | /** 68 | * Parse the action sample. 69 | * 70 | * @param space The space information class. 71 | * @param sample The parsed sample. 72 | */ 73 | void actionSample(const Space* space, arma::mat& sample); 74 | 75 | /** 76 | * Parse the info data. 77 | * 78 | * @param reward The reward information. 79 | * @param done The information whether task succeed or not. 80 | */ 81 | void info(double& reward, bool& done, std::string& info); 82 | 83 | /** 84 | * Parse the environment data. 85 | * 86 | * @param instance The instance identifier. 87 | */ 88 | void environment(std::string& instance); 89 | 90 | /** 91 | * Parse the url data. 92 | * 93 | * @param url The url. 94 | */ 95 | void url(std::string& url); 96 | private: 97 | //! Store results of the given json string in the row'th of the given 98 | //! matrix v. 99 | void vec(const pjson::value_variant_vec_t& vector, arma::mat& v); 100 | 101 | // void vec(pjson::value_variant_vec_t&, arma::mat& v); 102 | 103 | //! Store results of the given json string in the row'th of the given 104 | //! matrix v. 105 | void vec(const pjson::value_variant_vec_t& vector, std::vector& v); 106 | 107 | //! Store results of the given json string in the row'th of the given 108 | //! matrix v. 109 | void vec(const pjson::value_variant_vec_t& vector, std::vector& v); 110 | 111 | //! Locally-stored document to parse the json string. 112 | pjson::document doc; 113 | 114 | //! Locally-stored data stream. 115 | char* dataStream; 116 | }; 117 | 118 | } // namespace gym 119 | 120 | 121 | // Include implementation. 122 | #include "parser_impl.hpp" 123 | 124 | #endif 125 | -------------------------------------------------------------------------------- /cpp/environment_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file environment_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of miscellaneous environment routines. 6 | */ 7 | #ifndef GYM_ENVIRONMENT_IMPL_HPP 8 | #define GYM_ENVIRONMENT_IMPL_HPP 9 | 10 | // In case it hasn't been included yet. 11 | #include "environment.hpp" 12 | #include "messages.hpp" 13 | 14 | namespace gym { 15 | 16 | inline Environment::Environment() : renderValue(false) 17 | { 18 | // Nothing to do here. 19 | } 20 | 21 | inline Environment::Environment(const std::string& host, const std::string& port) : 22 | renderValue(false) 23 | { 24 | client.connect(host, port); 25 | } 26 | 27 | inline Environment::Environment( 28 | const std::string& host, 29 | const std::string& port, 30 | const std::string& environment) : 31 | renderValue(false) 32 | { 33 | client.connect(host, port); 34 | make(environment); 35 | } 36 | 37 | inline void Environment::make(const std::string& environment) 38 | { 39 | client.send(messages::EnvironmentName(environment)); 40 | 41 | std::string json; 42 | client.receive(json); 43 | parser.parse(json); 44 | parser.environment(instance); 45 | 46 | observationSpace(); 47 | actionSpace(); 48 | 49 | observation_space.client(client); 50 | action_space.client(client); 51 | record_episode_stats.client(client); 52 | } 53 | 54 | inline void Environment::render() 55 | { 56 | if (renderValue) 57 | { 58 | renderValue = false; 59 | } 60 | else 61 | { 62 | renderValue = true; 63 | } 64 | } 65 | 66 | inline void Environment::close() 67 | { 68 | client.send(messages::EnvironmentClose()); 69 | } 70 | 71 | inline const arma::mat& Environment::reset() 72 | { 73 | client.send(messages::EnvironmentReset()); 74 | 75 | std::string json; 76 | client.receive(json); 77 | 78 | parser.parse(json); 79 | parser.observation(&observation_space, observation); 80 | 81 | return observation; 82 | } 83 | 84 | inline void Environment::step(const arma::mat& action) 85 | { 86 | client.send(messages::Step(action, action_space, renderValue)); 87 | 88 | std::string json; 89 | client.receive(json); 90 | 91 | parser.parse(json); 92 | parser.observation(&observation_space, observation); 93 | parser.info(reward, done, info); 94 | } 95 | 96 | inline void Environment::seed(const size_t s) 97 | { 98 | client.send(messages::EnvironmentSeed(s)); 99 | } 100 | 101 | inline void Environment::compression(const size_t compression) 102 | { 103 | client.compression(compression); 104 | client.send(messages::ServerCompression(compression)); 105 | } 106 | 107 | inline void Environment::observationSpace() 108 | { 109 | client.send(messages::EnvironmentObservationSpace()); 110 | 111 | std::string json; 112 | client.receive(json); 113 | 114 | parser.parse(json); 115 | parser.space(&observation_space); 116 | } 117 | 118 | inline void Environment::actionSpace() 119 | { 120 | client.send(messages::EnvironmentActionSpace()); 121 | 122 | std::string json; 123 | client.receive(json); 124 | 125 | parser.parse(json); 126 | parser.space(&action_space); 127 | } 128 | 129 | inline std::string Environment::url() 130 | { 131 | client.send(messages::URL()); 132 | 133 | std::string json; 134 | client.receive(json); 135 | 136 | std::string url; 137 | parser.parse(json); 138 | parser.url(url); 139 | 140 | return url; 141 | } 142 | 143 | } // namespace gym 144 | 145 | #endif 146 | -------------------------------------------------------------------------------- /cpp/environment.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file environment.hpp 3 | * @author Marcus Edel 4 | * 5 | * Definition of miscellaneous environment routines. 6 | */ 7 | #ifndef GYM_ENVIRONMENT_HPP 8 | #define GYM_ENVIRONMENT_HPP 9 | 10 | #include 11 | #include 12 | 13 | #include "client.hpp" 14 | #include "parser.hpp" 15 | #include "space.hpp" 16 | #include "record_episode_statistics.hpp" 17 | 18 | namespace gym { 19 | 20 | /* 21 | * Definition of the Environment class. 22 | */ 23 | class Environment 24 | { 25 | public: 26 | //! Locally stored observation space instance. 27 | Space observation_space; 28 | 29 | //! Locally stored action space instance. 30 | Space action_space; 31 | 32 | //! Locally stored record episode statistics instance. 33 | RecordEpisodeStatistics record_episode_stats; 34 | 35 | //! Locally stored reward value. 36 | double reward; 37 | 38 | //! Locally stored info value. 39 | std::string info; 40 | 41 | //! Locally stored done value. 42 | bool done; 43 | 44 | //! Locally-stored observation object. 45 | arma::mat observation; 46 | 47 | //! Locally-stored instance identifier. 48 | std::string instance; 49 | 50 | /** 51 | * Instantiate the Environment object. 52 | */ 53 | Environment(); 54 | 55 | /* 56 | * Instantiate the Environment object using the specified parameter. 57 | * 58 | * @param host The host name used for the connection. 59 | * @param port The port used for the connection. 60 | */ 61 | Environment(const std::string& host, 62 | const std::string& port); 63 | 64 | /* 65 | * Instantiate the Environment object using the specified parameter. 66 | * 67 | * @param host The host name used for the connection. 68 | * @param port The port used for the connection. 69 | * @param environment Name of the environments used to train/evaluate 70 | * the model. 71 | */ 72 | Environment(const std::string& host, 73 | const std::string& port, 74 | const std::string& environment); 75 | 76 | /* 77 | * Instantiate the environment object using the specified environment name. 78 | * 79 | * @param environment Name of the environments used to train/evaluate 80 | * the model. 81 | */ 82 | void make(const std::string& environment); 83 | 84 | /* 85 | * Renders the environment. 86 | */ 87 | void render(); 88 | 89 | /* 90 | * Close the environment. 91 | */ 92 | void close(); 93 | 94 | /* 95 | * Resets the state of the environment and returns an initial observation. 96 | */ 97 | const arma::mat& reset(); 98 | 99 | /* 100 | * Run one timestep of the environment's dynamics using the specified action. 101 | * 102 | * @param action The action performed at the timestep. 103 | */ 104 | void step(const arma::mat& action); 105 | 106 | /* 107 | * Sets the seed for this env's random number generator. 108 | * 109 | * @param s The seed used for the random number generator. 110 | */ 111 | void seed(const size_t s); 112 | 113 | /* 114 | * Sets the compression level in range [0, 9] where 0 means no compression. 115 | * 116 | * @param compression The compression level. 117 | */ 118 | void compression(const size_t compression); 119 | 120 | /* 121 | * Get the environment url. 122 | */ 123 | std::string url(); 124 | 125 | private: 126 | //! Get the observation space information. 127 | void observationSpace(); 128 | 129 | //! The the action space information. 130 | void actionSpace(); 131 | 132 | //! Locally-stored client object. 133 | Client client; 134 | 135 | //! Locally-stored parser object. 136 | Parser parser; 137 | 138 | //! Locally-stored current render value. 139 | bool renderValue; 140 | }; 141 | 142 | } // namespace gym 143 | 144 | // Include implementation. 145 | #include "environment_impl.hpp" 146 | 147 | #endif 148 | -------------------------------------------------------------------------------- /cpp/messages.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file messages.hpp 3 | * @author Marcus Edel 4 | * 5 | * Miscellaneous messages. 6 | */ 7 | #ifndef GYM_MESSAGES_HPP 8 | #define GYM_MESSAGES_HPP 9 | 10 | #include 11 | #include 12 | 13 | #include "space.hpp" 14 | 15 | namespace gym { 16 | namespace messages { 17 | 18 | //! Create message to set the enviroment name. 19 | static inline std::string EnvironmentName(const std::string& name) 20 | { 21 | return "{\"env\":{\"name\": \"" + name + "\"}}"; 22 | } 23 | 24 | //! Create message to reset the enviroment. 25 | static inline std::string EnvironmentReset() 26 | { 27 | return "{\"env\":{\"action\": \"reset\"}}"; 28 | } 29 | 30 | //! Create message to start the Record Episode Statistics. 31 | static inline std::string RecordEpisodeStatisticsStart() 32 | { 33 | return "{\"record_episode_stats\":{\"action\": \"start\"}}"; 34 | } 35 | 36 | //! Create message to set the compression level. 37 | static inline std::string ServerCompression(const size_t compression) 38 | { 39 | return "{\"server\":{\"compression\": \"" + 40 | std::to_string(compression) + "\"}}"; 41 | } 42 | 43 | //! Create message to set the enviroment seed. 44 | static inline std::string EnvironmentSeed(const size_t seed) 45 | { 46 | return "{\"env\":{\"seed\": \"" + std::to_string(seed) + "\"}}"; 47 | } 48 | 49 | //! Create message to close the enviroment. 50 | static inline std::string EnvironmentClose() 51 | { 52 | return "{\"env\":{\"action\": \"close\"}}"; 53 | } 54 | 55 | //! Create message to get the action space. 56 | static inline std::string EnvironmentActionSpace() 57 | { 58 | return "{\"env\":{\"action\": \"actionspace\"}}"; 59 | } 60 | 61 | //! Create message to get the observation space. 62 | static inline std::string EnvironmentObservationSpace() 63 | { 64 | return "{\"env\":{\"action\": \"observationspace\"}}"; 65 | } 66 | 67 | //! Create message to get the action space sample. 68 | static inline std::string EnvironmentActionSpaceSample() 69 | { 70 | return "{\"env\":{\"actionspace\": \"sample\"}}"; 71 | } 72 | 73 | //! Create message to get the action space. 74 | static inline std::string Step( 75 | const arma::mat& action, Space& space, const bool render) 76 | { 77 | if (space.type == Space::DISCRETE) 78 | { 79 | arma::mat actionTmp = action; 80 | if (action.n_elem > 1) 81 | { 82 | actionTmp(0) = arma::as_scalar(arma::find(action.max() == action, 1)); 83 | } 84 | 85 | return "{\"step\":{\"action\":" + std::to_string((int) actionTmp(0)) + 86 | ", \"render\":" + std::to_string(render) + "}}"; 87 | } 88 | if (space.type == Space::MULTIDISCRETE) 89 | { 90 | std::string actionStr = "["; 91 | for (size_t i = 0; i < action.n_elem; ++i) 92 | { 93 | if (i < (action.n_elem - 1)) 94 | { 95 | actionStr += std::to_string((int) action(i)) + ","; 96 | } 97 | else 98 | { 99 | actionStr += std::to_string((int) action(i)); 100 | } 101 | } 102 | actionStr += "]"; 103 | 104 | std::string msg = "{\"step\":{\"action\":" + actionStr + 105 | ", \"render\":" + std::to_string(render) + "}}"; 106 | 107 | return msg; 108 | } 109 | if (space.type == Space::BOX) 110 | { 111 | std::string actionStr = "["; 112 | for (size_t i = 0; i < action.n_elem; ++i) 113 | { 114 | if (i < (action.n_elem - 1)) 115 | { 116 | actionStr += std::to_string((double) action(i)) + ","; 117 | } 118 | else 119 | { 120 | actionStr += std::to_string((double) action(i)); 121 | } 122 | } 123 | actionStr += "]"; 124 | 125 | std::string msg = "{\"step\":{\"action\":" + actionStr + 126 | ", \"render\":" + std::to_string(render) + "}}"; 127 | 128 | return msg; 129 | } 130 | return ""; 131 | } 132 | 133 | //! Create message to get the url. 134 | static inline std::string URL() 135 | { 136 | return "{\"url\":{\"action\": \"url\"}}"; 137 | } 138 | 139 | } // namespace messages 140 | } // namespace gym 141 | 142 | #endif 143 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gym-tcp-api 2 | 3 | This project provides a distributed infrastructure (TCP API) to the OpenAI Gym toolkit, allowing development in languages other than python. 4 | 5 | The server is written in elixir, enabling a distributed infrastructure. Where each node makes use of a limitted set of processes that can be used to perform time consuming tasks (4 python instances per default). 6 | 7 | ## Contents 8 | 9 | 1. [Dependencies](#dependencies) 10 | 2. [Installation](#installation) 11 | 3. [Getting started](#getting-started) 12 | 3. [Demo](#demo) 13 | 4. [Distributed Server](#distributed-server) 14 | 5. [API specification](#api-specification) 15 | 6. [FAQ](#faq) 16 | 17 | ## Dependencies 18 | 19 | The server has the following dependencies: 20 | 21 | Python3 22 | Elixir >= 1.0 23 | OpenAI Gym. 24 | 25 | The c++ example agent has the following dependencies: 26 | 27 | Armadillo >= 4.200.0 28 | Boost (system, thread, iostreams) 29 | CMake >= 2.8.5 30 | 31 | ## Installation 32 | 33 | ### Server 34 | 35 | First, checkout the repo and change into the unpacked directory: 36 | 37 | $ git clone https://github.com/zoq/gym_tcp_api.git 38 | $ cd gym_tcp_api 39 | 40 | Then install the elixir dependencies: 41 | 42 | $ mix deps.get 43 | 44 | ### C++ Client 45 | 46 | First, checkout the repo and change into the unpacked c++ client directory: 47 | 48 | $ git clone https://github.com/zoq/gym_tcp_api.git 49 | $ cd gym_tcp_api/cpp/ 50 | 51 | Then, make a build directory. The directory can have any name, not just 'build', but 'build' is sufficient. 52 | 53 | $ mkdir build 54 | $ cd build 55 | 56 | The next step is to run CMake to configure the project. Running CMake is the equivalent to running ./configure with autotools 57 | 58 | $ cmake ../ 59 | 60 | Once CMake is configured, building the example agent is as simple as typing 'make'. 61 | 62 | $ make 63 | 64 | ## Getting started 65 | 66 | To start the server from the command line: 67 | * For Python server run: 68 | 69 | $ python python/server.py 70 | 71 | * For Elixir server run: 72 | 73 | $ iex -S mix 74 | 75 | In a separate terminal, you can then run the example agent: 76 | 77 | $ ./example 78 | 79 | ## Demo 80 | 81 | The distributed demo server is reachable at ```kurg.org (port 4040```) and can be used for testing. Each node provides access to the Classic control and Atari environments. Note that each node has limited resources so that the response time might vary. If you record your algorithm's performance on an environment you can access the video and metadata at https://kurg.org/media/gym/ 82 | 83 |

84 | breakout sample image sequence breakout sample image sequence breakout sample image sequence 85 |

86 | 87 | ## Distributed Server 88 | 89 | First, we need to find out the IP addresses of both machines. In this case, the IP address is 192.168.0.103 for the first machine. On the other machine, the IP address is 192.168.0.104. 90 | 91 | In the first window: 92 | 93 | $ iex --name one@192.168.0.103 --cookie "gym" -S mix 94 | 95 | In the second window: 96 | 97 | $ iex --name two@192.168.0.104 --cookie "gym" -S mix 98 | 99 | In the interactive shell of the second node type: 100 | 101 | $ Node.connect :'one@192.168.0.103' 102 | 103 | ## API specification 104 | We use JSON as the format to cimmunicate with the server. 105 | 106 | Create the specified environment: 107 | 108 | {"env" {"name": "CartPole-v0"}} 109 | 110 | Close the environment: 111 | 112 | {"env" {"action": "close"}} 113 | 114 | Reset the state of the environment: 115 | 116 | {"env" {"action": "reset"}} 117 | 118 | Set the enviroment seed: 119 | 120 | {"env" {"seed": "3"}} 121 | 122 | Get the action space information: 123 | 124 | {"env" {"action": "actionspace"}} 125 | 126 | Get observation space information: 127 | 128 | {"env" {"action": "observationspace"}} 129 | 130 | Get action space sample: 131 | 132 | {"env" {"actionspace": "sample"}} 133 | 134 | Step though an environment using an action: 135 | 136 | {"step" {"action": "1"}} 137 | 138 | {"step" {"action": "[0, 1, 0, 0]"}} 139 | 140 | Start the record episode statistics: 141 | 142 | {"record_episode_stats" {"action": "start"}} 143 | 144 | ## FAQ 145 | 1. In the Erlang/OTP 21, erlport may not be compiled, because the latest version was not reflected in the official Erlport GitHub. 146 | 147 | - SOL) Change the version of erlport manually in mix.exs. As of October, 2018, the "0.10.0" version worked. See this https://github.com/hdima/erlport 148 | ``` 149 | defp deps do 150 | [ 151 | {:erlport, "~> 0.10.0"}, # Choose the version. 152 | {:poolboy, "~> 1.5"} 153 | ] 154 | end 155 | ``` 156 | 157 | 2. Failed to fetch record for 'hexpm/poolboy' from registry (using cache) 158 | 159 | - SOL) This was a cache problem, remove the cache file and try again. 160 | ```bash 161 | rm "~/.hex/cache.ets" 162 | ``` 163 | 164 | 3. TypeError: super() takes at least 1 argument (0 given) 165 | super().__init__((), np.int64) 166 | 167 | - SOL 1) Using Python3 VirtualEnv (Recommended). 168 | 169 | $ virtualenv -p python3 envname 170 | 171 | - SOL 2) Make Python3 as the default Python (since gym is built on Python3). 172 | 173 | $ sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10 174 | -------------------------------------------------------------------------------- /cpp/parser_impl.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file parser_impl.hpp 3 | * @author Marcus Edel 4 | * 5 | * Implementation of miscellaneous parser routines. 6 | */ 7 | #ifndef GYM_PARSER_IMPL_HPP 8 | #define GYM_PARSER_IMPL_HPP 9 | 10 | // In case it hasn't been included yet. 11 | #include "parser.hpp" 12 | #include "space.hpp" 13 | 14 | namespace gym { 15 | 16 | inline Parser::Parser() : dataStream(0) 17 | { 18 | // Nothing to do here. 19 | } 20 | 21 | inline Parser::~Parser() 22 | { 23 | if ((dataStream != NULL) && (dataStream[0] == '\0')) 24 | delete[] dataStream; 25 | } 26 | 27 | inline Parser::Parser(const std::string& data) 28 | { 29 | parse(data); 30 | } 31 | 32 | inline void Parser::parse(const std::string& data) 33 | { 34 | if ((dataStream != NULL) && (dataStream[0] == '\0')) 35 | delete[] dataStream; 36 | 37 | dataStream = new char[data.size() + 1]; 38 | std::copy(data.begin(), data.end(), dataStream); 39 | dataStream[data.size()] = '\0'; 40 | 41 | doc.deserialize_in_place(dataStream); 42 | } 43 | 44 | inline void Parser::actionSample(const Space* space, arma::mat& sample) 45 | { 46 | if (space->type == Space::DISCRETE) 47 | { 48 | if (sample.is_empty()) 49 | { 50 | sample = arma::mat(1, 1); 51 | } 52 | 53 | sample(0) = doc.get_object()[0].get_value().as_double(); 54 | } 55 | } 56 | 57 | inline void Parser::info(double& reward, bool& done, std::string& info) 58 | { 59 | const pjson::value_variant* doneValue = doc.find_value_variant("done"); 60 | if (doneValue != NULL) 61 | done = doneValue->as_bool(); 62 | 63 | const pjson::value_variant* rewardValue = doc.find_value_variant("reward"); 64 | if (rewardValue != NULL) 65 | reward = rewardValue->as_double(); 66 | } 67 | 68 | inline void Parser::environment(std::string& instance) 69 | { 70 | pjson::key_value_vec_t& obj = doc.get_object(); 71 | for (size_t i = 0u; i < obj.size(); ++i) 72 | { 73 | if (std::strncmp("instance", obj[i].get_key().m_p, 8) == 0) 74 | instance = obj[i].get_value().get_string_ptr(); 75 | } 76 | } 77 | 78 | inline void Parser::observation(const Space* space, arma::mat& observation) 79 | { 80 | const pjson::value_variant* value = doc.find_value_variant("observation"); 81 | if (space->boxShape.size() == 1) 82 | { 83 | observation = arma::mat(space->boxShape[0], 1); 84 | vec(value->get_array(), observation); 85 | } 86 | else if (space->boxShape.size() == 2) 87 | { 88 | observation = arma::mat(space->boxShape[1], space->boxShape[0]); 89 | 90 | size_t elem = 0; 91 | const pjson::value_variant_vec_t& array1 = value->get_array(); 92 | for (size_t i = 0; i < array1.size(); i++) 93 | { 94 | const pjson::value_variant_vec_t& array2 = array1[i].get_array(); 95 | for (size_t j = 0; j < array2.size(); j++) 96 | { 97 | observation(elem++) = array2[j].as_double(); 98 | } 99 | } 100 | 101 | observation = observation.t(); 102 | } 103 | else if (space->boxShape.size() == 3) 104 | { 105 | arma::cube temp(space->boxShape[1], space->boxShape[0], space->boxShape[2]); 106 | observation = arma::mat(space->boxShape[0] * space->boxShape[1], 107 | space->boxShape[2]); 108 | 109 | size_t elem = 0; 110 | const pjson::value_variant_vec_t& array1 = value->get_array(); 111 | for (size_t i = 0; i < array1.size(); i++) 112 | { 113 | const pjson::value_variant_vec_t& array2 = array1[i].get_array(); 114 | for (size_t j = 0; j < array2.size(); j++) 115 | { 116 | size_t z = 0; 117 | const pjson::value_variant_vec_t& array3 = array2[j].get_array(); 118 | for (size_t k = 0; k < array3.size(); k++, elem++, z++) 119 | { 120 | elem = elem % observation.n_rows; 121 | temp.slice(z)(elem) = array3[k].as_double(); 122 | } 123 | } 124 | } 125 | 126 | for (size_t i = 0; i < space->boxShape[2]; ++i) 127 | { 128 | arma::mat slice = arma::trans(temp.slice(i)); 129 | observation.col(i) = arma::vectorise(slice); 130 | } 131 | } 132 | } 133 | 134 | inline void Parser::space(Space* space) 135 | { 136 | const pjson::key_value_vec_t& obj = doc.find_value_variant( 137 | "info")->get_object(); 138 | for (size_t i = 0u; i < obj.size(); ++i) 139 | { 140 | if (std::strncmp("name", obj[i].get_key().m_p, 4) == 0) 141 | { 142 | if (std::strncmp("MultiDiscrete", 143 | obj[i].get_value().get_string_ptr(), 13) == 0) 144 | { 145 | space->type = Space::MULTIDISCRETE; 146 | } 147 | else if (std::strncmp("Discrete", 148 | obj[i].get_value().get_string_ptr(), 8) == 0) 149 | { 150 | space->type = Space::DISCRETE; 151 | } 152 | else if (std::strncmp("Box", 153 | obj[i].get_value().get_string_ptr(), 3) == 0) 154 | { 155 | space->type = Space::BOX; 156 | } 157 | } 158 | else if (std::strncmp("n", obj[i].get_key().m_p, 1) == 0) 159 | { 160 | space->n = obj[i].get_value().as_int64(); 161 | } 162 | else if (std::strncmp("high", obj[i].get_key().m_p, 4) == 0) 163 | { 164 | vec(obj[i].get_value().get_array(), space->boxHigh); 165 | } 166 | else if (std::strncmp("low", obj[i].get_key().m_p, 3) == 0) 167 | { 168 | vec(obj[i].get_value().get_array(), space->boxLow); 169 | } 170 | else if (std::strncmp("shape", obj[i].get_key().m_p, 5) == 0) 171 | { 172 | vec(obj[i].get_value().get_array(), space->boxShape); 173 | } 174 | } 175 | } 176 | 177 | inline void Parser::vec(const pjson::value_variant_vec_t& vector, arma::mat& v) 178 | { 179 | size_t idx = 0; 180 | for (pjson::uint i = 0; i < vector.size(); ++i) 181 | v(idx++) = vector[i].as_double(); 182 | } 183 | 184 | inline void Parser::vec( 185 | const pjson::value_variant_vec_t& vector, std::vector& v) 186 | { 187 | for (pjson::uint i = 0; i < vector.size(); ++i) 188 | v.push_back(vector[i].as_float()); 189 | } 190 | 191 | inline void Parser::vec( 192 | const pjson::value_variant_vec_t& vector, std::vector& v) 193 | { 194 | for (pjson::uint i = 0; i < vector.size(); ++i) 195 | v.push_back(vector[i].as_int64()); 196 | } 197 | 198 | inline void Parser::url(std::string& url) 199 | { 200 | pjson::key_value_vec_t& obj = doc.get_object(); 201 | for (size_t i = 0u; i < obj.size(); ++i) 202 | { 203 | if (std::strncmp("url", obj[i].get_key().m_p, 3) == 0) 204 | url = obj[i].get_value().get_string_ptr(); 205 | } 206 | } 207 | 208 | } // namespace gym 209 | 210 | #endif 211 | -------------------------------------------------------------------------------- /cpp/client.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * @file client.hpp 3 | * @author Marcus Edel 4 | * 5 | * Miscellaneous client routines. 6 | */ 7 | #ifndef GYM_CLIENT_HPP 8 | #define GYM_CLIENT_HPP 9 | 10 | #include 11 | #include 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | namespace gym { 23 | 24 | using boost::asio::ip::tcp; 25 | using boost::asio::deadline_timer; 26 | using boost::lambda::_1; 27 | using boost::lambda::var; 28 | using boost::lambda::bind; 29 | 30 | /** 31 | * Implementation of the Client. 32 | */ 33 | class Client 34 | { 35 | public: 36 | static void async_read_handler(const boost::system::error_code& err, 37 | boost::system::error_code* err_out, 38 | std::size_t bytes_transferred, 39 | std::size_t* bytes_out) 40 | { 41 | *err_out = err; 42 | *bytes_out = bytes_transferred; 43 | } 44 | 45 | /** 46 | * Create the Client object using the given host and port. 47 | * 48 | * @param host The hostname to connect. 49 | * @param port The port used for the connection. 50 | */ 51 | Client() : 52 | s(io_context), 53 | deadline(io_context), 54 | compressionLevel(0) 55 | { 56 | deadline.expires_at(boost::posix_time::pos_infin); 57 | 58 | // Start the persistent actor that checks for deadline expiry. 59 | check_deadline(); 60 | } 61 | 62 | ~Client() 63 | { 64 | // The socket is closed so that any outstanding asynchronous operations are 65 | // cancelled. This allows the blocked connect(), read_line() or write_line() 66 | // functions to return. 67 | boost::system::error_code ignored_ec; 68 | s.close(ignored_ec); 69 | } 70 | 71 | void connect(const std::string& host, const std::string& port) 72 | { 73 | tcp::resolver resolver(io_context); 74 | tcp::resolver::results_type iterator = resolver.resolve(tcp::v4(), host, port); 75 | 76 | // Set a deadline for the asynchronous operation. 77 | deadline.expires_from_now(boost::posix_time::seconds(10)); 78 | 79 | // Set up the variable that receives the result of the asynchronous 80 | // operation. 81 | boost::system::error_code ec = boost::asio::error::would_block; 82 | 83 | // Start the asynchronous operation. 84 | boost::asio::async_connect(s, iterator, var(ec) = _1); 85 | 86 | // Block until the asynchronous operation has completed. 87 | do io_context.run_one(); while (ec == boost::asio::error::would_block); 88 | 89 | // Determine whether a connection was successfully established. 90 | if (ec || !s.is_open()) 91 | { 92 | throw boost::system::system_error( 93 | ec ? ec : boost::asio::error::operation_aborted); 94 | } 95 | } 96 | 97 | /** 98 | * Receive a message using the currently open socket. 99 | * 100 | * @param data The received data. 101 | */ 102 | void receive(std::string& data) 103 | { 104 | // Set a deadline for the asynchronous operation. 105 | deadline.expires_from_now(boost::posix_time::seconds(10)); 106 | 107 | // Set up the variable that receives the result of the asynchronous 108 | // operation. 109 | boost::system::error_code ec = boost::asio::error::would_block; 110 | 111 | boost::asio::streambuf response; 112 | size_t reply_length; 113 | 114 | boost::asio::async_read_until(s, response, "\r\n\r\n", 115 | boost::bind(async_read_handler, boost::asio::placeholders::error, &ec, 116 | boost::asio::placeholders::bytes_transferred, &reply_length)); 117 | 118 | // Block until the asynchronous operation has completed. 119 | do io_context.run_one(); while (ec == boost::asio::error::would_block); 120 | 121 | if (ec) 122 | { 123 | throw boost::system::system_error(ec); 124 | } 125 | 126 | data = std::string( 127 | boost::asio::buffers_begin(response.data()), 128 | boost::asio::buffers_begin(response.data()) + reply_length); 129 | 130 | if (compressionLevel > 0) 131 | { 132 | boost::iostreams::filtering_streambuf in; 133 | in.push( boost::iostreams::zlib_decompressor()); 134 | in.push(boost::iostreams::array_source(data.data(), data.size())); 135 | 136 | std::stringstream ss; 137 | boost::iostreams::copy(in, ss); 138 | data = ss.str(); 139 | } 140 | } 141 | 142 | /** 143 | * Send a message using the currently open socket. 144 | * 145 | * @param data The data to be send. 146 | */ 147 | void send(const std::string& data) 148 | { 149 | // Set a deadline for the asynchronous operation. 150 | deadline.expires_from_now(boost::posix_time::seconds(10)); 151 | 152 | // Set up the variable that receives the result of the asynchronous 153 | // operation. 154 | boost::system::error_code ec = boost::asio::error::would_block; 155 | 156 | std::string d = data + "\r\n"; 157 | 158 | boost::asio::async_write(s, boost::asio::buffer(d), 159 | boost::asio::transfer_exactly(d.size()), var(ec) = _1); 160 | 161 | // Block until the asynchronous operation has completed. 162 | do io_context.run_one(); while (ec == boost::asio::error::would_block); 163 | 164 | if (ec) 165 | { 166 | throw boost::system::system_error(ec); 167 | } 168 | } 169 | 170 | /* 171 | * The compression level in range [0, 9] where 0 means no compression used for 172 | * receiving data. 173 | * 174 | * @param compression The compression level. 175 | */ 176 | void compression(const size_t compression) 177 | { 178 | compressionLevel = compression; 179 | } 180 | 181 | private: 182 | void check_deadline() 183 | { 184 | // Check whether the deadline has passed. We compare the deadline against 185 | // the current time since a new asynchronous operation may have moved the 186 | // deadline before this actor had a chance to run. 187 | if (deadline.expires_at() <= deadline_timer::traits_type::now()) 188 | { 189 | // There is no longer an active deadline. The expiry is set to positive 190 | // infinity so that the actor takes no action until a new deadline is set. 191 | deadline.expires_at(boost::posix_time::pos_infin); 192 | } 193 | 194 | // Put the actor back to sleep. 195 | deadline.async_wait(bind(&Client::check_deadline, this)); 196 | } 197 | 198 | //! Locally stored io service. 199 | boost::asio::io_context io_context; 200 | 201 | //! Locally stored socket object. 202 | tcp::socket s; 203 | 204 | //! Object to control connection timeouts. 205 | deadline_timer deadline; 206 | 207 | //! Locally-stored compression parameter. 208 | size_t compressionLevel; 209 | }; // class Client 210 | 211 | } // namespace gym 212 | 213 | #endif 214 | -------------------------------------------------------------------------------- /priv/worker.py: -------------------------------------------------------------------------------- 1 | """ 2 | @file worker.py 3 | @author Marcus Edel 4 | @author Mehul Kumar Nirala 5 | @author Nabanita Dash 6 | Container and manager for the environments instantiated on this server. 7 | """ 8 | 9 | import erlport 10 | from erlport import erlang 11 | 12 | import json 13 | import uuid 14 | import numpy as np 15 | 16 | import gym 17 | from gym.wrappers import RecordEpisodeStatistics 18 | 19 | try: 20 | import zlib 21 | except ImportError: 22 | pass 23 | 24 | import logging 25 | logger = logging.getLogger(__name__) 26 | logger.setLevel(logging.INFO) 27 | 28 | 29 | try: 30 | unicode = unicode 31 | except NameError: 32 | # 'unicode' is undefined, must be Python 3 33 | str = str 34 | unicode = str 35 | bytes = bytes 36 | basestring = (str,bytes) 37 | else: 38 | # 'unicode' exists, must be Python 2 39 | str = str 40 | unicode = unicode 41 | bytes = str 42 | basestring = basestring 43 | 44 | """ 45 | Container and manager for the environments instantiated 46 | on this server. The Envs class is based on the gym-http-api project 47 | 48 | @misc{gymhttpapi2016, 49 | title = {OpenAI gym-http-api}, 50 | year = {2016}, 51 | publisher = {GitHub}, 52 | journal = {GitHub repository}, 53 | howpublished = {https://github.com/openai/gym-http-api} 54 | } 55 | """ 56 | class Envs(object): 57 | def __init__(self): 58 | self.envs = {} 59 | self.id_len = 13 60 | 61 | def _lookup_env(self, instance_id): 62 | try: 63 | return self.envs[instance_id] 64 | except KeyError: 65 | return None 66 | 67 | def _remove_env(self, instance_id): 68 | try: 69 | del self.envs[instance_id] 70 | except KeyError: 71 | raise InvalidUsage('Instance_id {} unknown'.format(instance_id)) 72 | 73 | def create(self, env_id): 74 | try: 75 | env = gym.make(env_id) 76 | except gym.error.Error: 77 | raise InvalidUsage( 78 | "Attempted to look up malformed environment ID '{}'".format(env_id)) 79 | 80 | instance_id = str(uuid.uuid4().hex)[:self.id_len] 81 | self.envs[instance_id] = env 82 | return instance_id 83 | 84 | def reset(self, instance_id): 85 | env = self._lookup_env(instance_id) 86 | obs = env.reset() 87 | return env.observation_space.to_jsonable(obs) 88 | 89 | def step(self, instance_id, action, render): 90 | env = self._lookup_env(instance_id) 91 | action_from_json = env.action_space.from_jsonable(action) 92 | if (not isinstance(action_from_json, (list))): 93 | action_from_json = int(action_from_json) 94 | 95 | if render: env.render() 96 | [observation, reward, done, info] = env.step(action_from_json) 97 | 98 | obs_jsonable = env.observation_space.to_jsonable(observation) 99 | return [obs_jsonable, reward, done, info] 100 | 101 | def seed(self, s): 102 | env = self._lookup_env(instance_id) 103 | env.seed(int(s)) 104 | 105 | def get_action_space_info(self, instance_id): 106 | env = self._lookup_env(instance_id) 107 | return self._get_space_properties(env.action_space) 108 | 109 | def get_action_space_sample(self, instance_id): 110 | env = self._lookup_env(instance_id) 111 | return env.action_space.sample() 112 | 113 | def get_observation_space_info(self, instance_id): 114 | env = self._lookup_env(instance_id) 115 | return self._get_space_properties(env.observation_space) 116 | 117 | def _get_space_properties(self, space): 118 | info = {} 119 | info['name'] = space.__class__.__name__ 120 | if info['name'] == 'Discrete': 121 | info['n'] = space.n 122 | elif info['name'] == 'Box': 123 | info['shape'] = space.shape 124 | # It's not JSON compliant to have Infinity, -Infinity, NaN. 125 | # Many newer JSON parsers allow it, but many don't. Notably python json 126 | # module can read and write such floats. So we only here fix 127 | # "export version", also make it flat. 128 | info['low'] = [(x if x != -np.inf else -1e100) for x 129 | in np.array(space.low ).flatten()] 130 | info['high'] = [(x if x != +np.inf else +1e100) for x 131 | in np.array(space.high).flatten()] 132 | elif info['name'] == 'HighLow': 133 | info['num_rows'] = space.num_rows 134 | info['matrix'] = [((float(x) if x != -np.inf else -1e100) if x != +np.inf 135 | else +1e100) for x in np.array(space.matrix).flatten()] 136 | elif info['name'] == 'MultiDiscrete': 137 | info['n'] = space.num_discrete_space 138 | info['low'] = [(x if x != -np.inf else -1e100) for x 139 | in np.array(space.low ).flatten()] 140 | info['high'] = [(x if x != +np.inf else +1e100) for x 141 | in np.array(space.high).flatten()] 142 | return info 143 | 144 | def record_episode_stats(self, instance_id): 145 | env = self._lookup_env(instance_id) 146 | self.envs[instance_id] = RecordEpisodeStatistics(env) 147 | 148 | def env_close(self, instance_id): 149 | env = self._lookup_env(instance_id) 150 | 151 | if env != None: 152 | env.close() 153 | self._remove_env(instance_id) 154 | 155 | def env_close_all(self): 156 | for key in list(self.envs.keys()): 157 | self.env_close(key) 158 | 159 | """ 160 | Error handling. 161 | """ 162 | class InvalidUsage(Exception): 163 | status_code = 400 164 | def __init__(self, message, status_code=None, payload=None): 165 | Exception.__init__(self) 166 | self.message = message 167 | if status_code is not None: 168 | self.status_code = status_code 169 | self.payload = payload 170 | 171 | def to_dict(self): 172 | rv = dict(self.payload or ()) 173 | rv['message'] = self.message 174 | return rv 175 | 176 | """ 177 | Json does not support float32/int64 serialization 178 | The following class converts to json defaults. 179 | So that json.dumps can serialize the dict to json. 180 | """ 181 | class NDArrayEncoder(json.JSONEncoder): 182 | def default(self, obj): 183 | if isinstance(obj, np.ndarray): 184 | return obj.tolist() 185 | try: 186 | return json.JSONEncoder.default(self, obj) 187 | except: 188 | return obj.tolist() 189 | 190 | 191 | """ 192 | Parse parameters. 193 | """ 194 | def get_optional_params(json, param1, param2): 195 | if (param1 in json): 196 | if param2 in json[param1]: 197 | return json[param1][param2] 198 | else: 199 | return None 200 | else: 201 | return None 202 | 203 | def get_optional_param(json, param): 204 | if (param in json): 205 | return json[param] 206 | else: 207 | return None 208 | 209 | envs = Envs() 210 | enviroment = None 211 | instance_id = None 212 | close = True 213 | compressionLevel = 0 214 | 215 | def process_data(data, level = 0): 216 | if level > 0: 217 | return zlib.compress(data.encode(), level) + b"\r\n\r\n" 218 | 219 | return data + "\r\n\r\n" 220 | 221 | """ 222 | Handle the incoming reponses. 223 | """ 224 | def process_response(response): 225 | global enviroment 226 | global instance_id 227 | global close 228 | global envs 229 | global compressionLevel 230 | 231 | response = unicode(response, "utf-8") 232 | data = response.strip() 233 | 234 | if (len(data) == 0): 235 | envs.env_close_all() 236 | return process_data("error", compressionLevel) 237 | 238 | jsonMessage = json.loads(data) 239 | 240 | enviroment = get_optional_params(jsonMessage, "env", "name") 241 | if isinstance(enviroment, basestring): 242 | compressionLevel = 0 243 | if instance_id != None: 244 | envs.env_close(instance_id) 245 | 246 | instance_id = envs.create(enviroment) 247 | data = json.dumps({"instance" : instance_id}, cls = NDArrayEncoder) 248 | return process_data(data, compressionLevel) 249 | 250 | compression = get_optional_params(jsonMessage, "server", "compression") 251 | if isinstance(compression, basestring): 252 | try: 253 | compressionLevel = int(compression) 254 | except ValueError: 255 | compressionLevel = 0 256 | close = True 257 | 258 | actionspace = get_optional_params(jsonMessage, "env", "actionspace") 259 | if isinstance(actionspace, basestring): 260 | if actionspace == "sample": 261 | sample = envs.get_action_space_sample(instance_id) 262 | 263 | data = json.dumps({"sample" : sample}, cls = NDArrayEncoder) 264 | return process_data(data, compressionLevel) 265 | 266 | envAction = get_optional_params(jsonMessage, "env", "action") 267 | if isinstance(envAction, basestring): 268 | if envAction == "close": 269 | envs.env_close(instance_id) 270 | close = False 271 | return "" 272 | elif envAction == "reset": 273 | observation = envs.reset(instance_id) 274 | data = json.dumps({"observation" : observation}, cls = NDArrayEncoder) 275 | return process_data(data, compressionLevel) 276 | elif envAction == "actionspace": 277 | info = envs.get_action_space_info(instance_id) 278 | data = json.dumps({"info" : info}, cls = NDArrayEncoder) 279 | return process_data(data, compressionLevel) 280 | elif envAction == "observationspace": 281 | info = envs.get_observation_space_info(instance_id) 282 | data = json.dumps({"info" : info}, cls = NDArrayEncoder) 283 | return process_data(data, compressionLevel) 284 | 285 | step = get_optional_param(jsonMessage, "step") 286 | if step is not None: 287 | action = get_optional_param(jsonMessage["step"], "action") 288 | render = get_optional_param(jsonMessage["step"], "render") 289 | 290 | render = True if (render is not None and render == 1) else False 291 | 292 | [obs, reward, done, info] = envs.step( 293 | instance_id, action, render) 294 | 295 | data = json.dumps({"observation" : obs, 296 | "reward" : reward, 297 | "done" : done, 298 | "info" : info}, cls = NDArrayEncoder) 299 | return process_data(data, compressionLevel) 300 | 301 | seed = get_optional_params(jsonMessage, "env", "seed") 302 | if isinstance(seed, basestring): 303 | envs.seed(seed) 304 | 305 | record_episode_stats = get_optional_param(jsonMessage, "record_episode_stats") 306 | if record_episode_stats is not None: 307 | action = get_optional_param(jsonMessage["record_episode_stats"], "action") 308 | 309 | if action == "start": 310 | envs.record_episode_stats(instance_id) 311 | 312 | return "" 313 | -------------------------------------------------------------------------------- /python/server.py: -------------------------------------------------------------------------------- 1 | import json 2 | import uuid 3 | import numpy as np 4 | import socket 5 | import os 6 | from _thread import * 7 | import glob 8 | 9 | import gym 10 | from gym.wrappers import RecordEpisodeStatistics 11 | 12 | try: 13 | import zlib 14 | except ImportError: 15 | pass 16 | 17 | import logging 18 | logger = logging.getLogger(__name__) 19 | logger.setLevel(logging.INFO) 20 | 21 | 22 | ServerSocket = socket.socket() 23 | ServerSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 24 | 25 | host = '127.0.0.1' 26 | port = 4040 27 | ThreadCount = 0 28 | try: 29 | ServerSocket.bind((host, port)) 30 | except socket.error as e: 31 | print(str(e)) 32 | 33 | print('Waitiing for a Connection..') 34 | ServerSocket.listen(5) 35 | 36 | 37 | try: 38 | unicode = unicode 39 | except NameError: 40 | # 'unicode' is undefined, must be Python 3 41 | str = str 42 | unicode = str 43 | bytes = bytes 44 | basestring = (str,bytes) 45 | else: 46 | # 'unicode' exists, must be Python 2 47 | str = str 48 | unicode = unicode 49 | bytes = str 50 | basestring = basestring 51 | 52 | """ 53 | Container and manager for the environments instantiated 54 | on this server. The Envs class is based on the gym-http-api project 55 | @misc{gymhttpapi2016, 56 | title = {OpenAI gym-http-api}, 57 | year = {2016}, 58 | publisher = {GitHub}, 59 | journal = {GitHub repository}, 60 | howpublished = {https://github.com/openai/gym-http-api} 61 | } 62 | """ 63 | class Envs(object): 64 | def __init__(self): 65 | self.envs = {} 66 | self.id_len = 13 67 | 68 | def _lookup_env(self, instance_id): 69 | try: 70 | return self.envs[instance_id] 71 | except KeyError: 72 | return None 73 | 74 | def _remove_env(self, instance_id): 75 | try: 76 | del self.envs[instance_id] 77 | except KeyError: 78 | raise InvalidUsage('Instance_id {} unknown'.format(instance_id)) 79 | 80 | def create(self, env_id): 81 | try: 82 | env = gym.make(env_id) 83 | except gym.error.Error: 84 | raise InvalidUsage( 85 | "Attempted to look up malformed environment ID '{}'".format(env_id)) 86 | 87 | instance_id = str(uuid.uuid4().hex)[:self.id_len] 88 | self.envs[instance_id] = env 89 | return instance_id 90 | 91 | def reset(self, instance_id): 92 | env = self._lookup_env(instance_id) 93 | obs = env.reset() 94 | return env.observation_space.to_jsonable(obs) 95 | 96 | def step(self, instance_id, action, render): 97 | env = self._lookup_env(instance_id) 98 | action_from_json = env.action_space.from_jsonable(action) 99 | if (not isinstance(action_from_json, (list))): 100 | action_from_json = int(action_from_json) 101 | 102 | if render: env.render() 103 | [observation, reward, done, info] = env.step(action_from_json) 104 | 105 | obs_jsonable = env.observation_space.to_jsonable(observation) 106 | return [obs_jsonable, reward, done, info] 107 | 108 | def seed(self, s): 109 | env = self._lookup_env(instance_id) 110 | env.seed(int(s)) 111 | 112 | def get_action_space_info(self, instance_id): 113 | env = self._lookup_env(instance_id) 114 | return self._get_space_properties(env.action_space) 115 | 116 | def get_action_space_sample(self, instance_id): 117 | env = self._lookup_env(instance_id) 118 | return env.action_space.sample() 119 | 120 | def get_observation_space_info(self, instance_id): 121 | env = self._lookup_env(instance_id) 122 | return self._get_space_properties(env.observation_space) 123 | 124 | def _get_space_properties(self, space): 125 | info = {} 126 | info['name'] = space.__class__.__name__ 127 | if info['name'] == 'Discrete': 128 | info['n'] = space.n 129 | elif info['name'] == 'Box': 130 | info['shape'] = space.shape 131 | # It's not JSON compliant to have Infinity, -Infinity, NaN. 132 | # Many newer JSON parsers allow it, but many don't. Notably python json 133 | # module can read and write such floats. So we only here fix 134 | # "export version", also make it flat. 135 | info['low'] = [(x if x != -np.inf else -1e100) for x 136 | in np.array(space.low ).flatten()] 137 | info['high'] = [(x if x != +np.inf else +1e100) for x 138 | in np.array(space.high).flatten()] 139 | elif info['name'] == 'HighLow': 140 | info['num_rows'] = space.num_rows 141 | info['matrix'] = [((float(x) if x != -np.inf else -1e100) if x != +np.inf 142 | else +1e100) for x in np.array(space.matrix).flatten()] 143 | elif info['name'] == 'MultiDiscrete': 144 | info['n'] = space.num_discrete_space 145 | info['low'] = [(x if x != -np.inf else -1e100) for x 146 | in np.array(space.low ).flatten()] 147 | info['high'] = [(x if x != +np.inf else +1e100) for x 148 | in np.array(space.high).flatten()] 149 | return info 150 | 151 | def record_episode_stats(self, instance_id): 152 | env = self._lookup_env(instance_id) 153 | self.envs[instance_id] = RecordEpisodeStatistics(env) 154 | 155 | def env_close(self, instance_id): 156 | env = self._lookup_env(instance_id) 157 | 158 | if env != None: 159 | env.close() 160 | self._remove_env(instance_id) 161 | 162 | def env_close_all(self): 163 | for key in list(self.envs.keys()): 164 | self.env_close(key) 165 | 166 | """ 167 | Error handling. 168 | """ 169 | class InvalidUsage(Exception): 170 | status_code = 400 171 | def __init__(self, message, status_code=None, payload=None): 172 | Exception.__init__(self) 173 | self.message = message 174 | if status_code is not None: 175 | self.status_code = status_code 176 | self.payload = payload 177 | 178 | def to_dict(self): 179 | rv = dict(self.payload or ()) 180 | rv['message'] = self.message 181 | return rv 182 | 183 | """ 184 | Json does not support float32/int64 serialization 185 | The following class converts to json defaults. 186 | So that json.dumps can serialize the dict to json. 187 | """ 188 | class NDArrayEncoder(json.JSONEncoder): 189 | def default(self, obj): 190 | if isinstance(obj, np.ndarray): 191 | return obj.tolist() 192 | try: 193 | return json.JSONEncoder.default(self, obj) 194 | except: 195 | return obj.tolist() 196 | 197 | 198 | """ 199 | Parse parameters. 200 | """ 201 | def get_optional_params(json, param1, param2): 202 | if (param1 in json): 203 | if param2 in json[param1]: 204 | return json[param1][param2] 205 | else: 206 | return None 207 | else: 208 | return None 209 | 210 | def get_optional_param(json, param): 211 | if (param in json): 212 | return json[param] 213 | else: 214 | return None 215 | 216 | 217 | def process_data(data, level = 0): 218 | if level > 0: 219 | return zlib.compress(data.encode(), level) + b"\r\n\r\n" 220 | 221 | return str.encode(data + "\r\n\r\n") 222 | 223 | 224 | def recv_client(connection): 225 | data_len = 0 226 | buffer = '' 227 | while 1: 228 | try: 229 | data = connection.recv(1).decode("utf-8") 230 | except: 231 | return "" 232 | buffer += data 233 | data_len = data_len + 1 234 | if data_len >= 2 and buffer[data_len - 2:data_len] == '\r\n': 235 | break 236 | return buffer 237 | 238 | def threaded_client(connection): 239 | #connection.send(str.encode('Welcome to the Server\n')) 240 | envs = Envs() 241 | enviroment = None 242 | instance_id = None 243 | close = True 244 | compressionLevel = 0 245 | connection.settimeout(60 * 20) 246 | 247 | """ 248 | Handle the incoming reponses. 249 | """ 250 | def process_response(response, connection, envs, enviroment, instance_id, close, compressionLevel): 251 | print("process") 252 | 253 | print(instance_id) 254 | 255 | #response = unicode(response, "utf-8") 256 | data = response.strip() 257 | 258 | if (len(data) == 0): 259 | envs.env_close_all() 260 | connection.send(str.encode(process_data("error", compressionLevel))) 261 | return enviroment, instance_id, close, compressionLevel 262 | 263 | jsonMessage = json.loads(data) 264 | print(jsonMessage) 265 | 266 | enviroment = get_optional_params(jsonMessage, "env", "name") 267 | if isinstance(enviroment, basestring): 268 | compressionLevel = 0 269 | if instance_id != None: 270 | envs.env_close(instance_id) 271 | 272 | instance_id = envs.create(enviroment) 273 | data = json.dumps({"instance" : instance_id}, cls = NDArrayEncoder) 274 | connection.send(process_data(data, compressionLevel)) 275 | return enviroment, instance_id, close, compressionLevel 276 | 277 | compression = get_optional_params(jsonMessage, "server", "compression") 278 | if isinstance(compression, basestring): 279 | try: 280 | compressionLevel = int(compression) 281 | except ValueError: 282 | compressionLevel = 0 283 | close = True 284 | 285 | actionspace = get_optional_params(jsonMessage, "env", "actionspace") 286 | if isinstance(actionspace, basestring): 287 | if actionspace == "sample": 288 | sample = envs.get_action_space_sample(instance_id) 289 | 290 | data = json.dumps({"sample" : sample}, cls = NDArrayEncoder) 291 | connection.send(process_data(data, compressionLevel)) 292 | return enviroment, instance_id, close, compressionLevel 293 | 294 | envAction = get_optional_params(jsonMessage, "env", "action") 295 | if isinstance(envAction, basestring): 296 | if envAction == "close": 297 | envs.env_close(instance_id) 298 | close = False 299 | connection.send(str.encode("")) 300 | return enviroment, instance_id, close, compressionLevel 301 | elif envAction == "reset": 302 | observation = envs.reset(instance_id) 303 | data = json.dumps({"observation" : observation}, cls = NDArrayEncoder) 304 | connection.send(process_data(data, compressionLevel)) 305 | return enviroment, instance_id, close, compressionLevel 306 | elif envAction == "actionspace": 307 | info = envs.get_action_space_info(instance_id) 308 | data = json.dumps({"info" : info}, cls = NDArrayEncoder) 309 | connection.send(process_data(data, compressionLevel)) 310 | return enviroment, instance_id, close, compressionLevel 311 | elif envAction == "observationspace": 312 | info = envs.get_observation_space_info(instance_id) 313 | data = json.dumps({"info" : info}, cls = NDArrayEncoder) 314 | connection.send(process_data(data, compressionLevel)) 315 | return enviroment, instance_id, close, compressionLevel 316 | 317 | step = get_optional_param(jsonMessage, "step") 318 | if step is not None: 319 | action = get_optional_param(jsonMessage["step"], "action") 320 | render = get_optional_param(jsonMessage["step"], "render") 321 | 322 | render = True if (render is not None and render == 1) else False 323 | 324 | [obs, reward, done, info] = envs.step( 325 | instance_id, action, render) 326 | 327 | data = json.dumps({"observation" : obs, 328 | "reward" : reward, 329 | "done" : done, 330 | "info" : info}, cls = NDArrayEncoder) 331 | connection.send(process_data(data, compressionLevel)) 332 | return enviroment, instance_id, close, compressionLevel 333 | 334 | seed = get_optional_params(jsonMessage, "env", "seed") 335 | if isinstance(seed, basestring): 336 | envs.seed(seed) 337 | 338 | url = get_optional_param(jsonMessage, "url") 339 | if url is not None: 340 | files = glob.glob("/var/log/gym/" + instance_id + "/*.mp4") 341 | if len(files) > 0: 342 | print(files) 343 | os.system("ffmpeg -y -i " + files[0] + " -c:v libvpx -crf 10 -b:v 1M -c:a libvorbis /var/log/gym/" + instance_id + "/output.webm") 344 | 345 | data = json.dumps({"url" : "https://gym.kurg.org/" + instance_id + "/output.webm"}, cls = NDArrayEncoder) 346 | connection.send(process_data(data, compressionLevel)) 347 | enviroment, instance_id, close, compressionLevel 348 | 349 | record_episode_stats = get_optional_param(jsonMessage, "record_episode_stats") 350 | if record_episode_stats is not None: 351 | action = get_optional_param(jsonMessage["record_episode_stats"], "action") 352 | 353 | if action == "start": 354 | envs.record_episode_stats(instance_id) 355 | 356 | connection.send(str.encode("")) 357 | return enviroment, instance_id, close, compressionLevel 358 | 359 | try: 360 | while True: 361 | buffer = recv_client(connection) 362 | if len(buffer) == 0: 363 | return 364 | enviroment, instance_id, close, compressionLevel = process_response(buffer, connection, envs, enviroment, instance_id, close, compressionLevel) 365 | except: 366 | #connection.close() 367 | return 368 | connection.close() 369 | 370 | while True: 371 | Client, address = ServerSocket.accept() 372 | print('Connected to: ' + address[0] + ':' + str(address[1])) 373 | start_new_thread(threaded_client, (Client, )) 374 | ThreadCount += 1 375 | print('Thread Number: ' + str(ThreadCount)) 376 | ServerSocket.close() -------------------------------------------------------------------------------- /cpp/pjson/pjson.h: -------------------------------------------------------------------------------- 1 | // File: pjson.h - written by Rich Geldreich 2012 - License: Unlicense http://unlicense.org/ 2 | #ifndef PURPLE_JSON_H 3 | #define PURPLE_JSON_H 4 | 5 | #ifdef WIN32 6 | #pragma once 7 | #endif 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | // ---- Macros 16 | 17 | #define PJSON_ASSERT assert 18 | #ifdef _MSC_VER 19 | #define PJSON_FORCEINLINE __forceinline 20 | #else 21 | #define PJSON_FORCEINLINE __attribute__((always_inline)) 22 | #endif 23 | 24 | #define PJSON_PARSE_STATS 0 25 | 26 | #define PJSON_DEFAULT_MIN_CHUNK_SIZE 4096 27 | #define PJSON_MAX_CHUNK_GROW_SIZE 8*1024*1024 28 | #define PJSON_DEFAULT_MAX_BYTES_TO_PRESERVE_ACROSS_RESETS 16*1024*1024 29 | 30 | #define PJSON_MIN(a, b) (((a) < (b)) ? (a) : (b)) 31 | #define PJSON_MAX(a, b) (((a) < (b)) ? (b) : (a)) 32 | 33 | namespace pjson 34 | { 35 | // ---- Types 36 | typedef unsigned char uint8; 37 | typedef unsigned int uint; 38 | typedef signed int int32; 39 | typedef unsigned int uint32; 40 | 41 | typedef int64_t int64; 42 | typedef uint64_t uint64; 43 | 44 | class document; 45 | class value_variant; 46 | struct value_variant_data; 47 | struct key_value_t; 48 | 49 | typedef std::vector char_vec_t; 50 | typedef std::string string_t; 51 | 52 | // Memory allocation 53 | 54 | inline void* pjson_malloc(size_t size) { return malloc(size); } 55 | inline void* pjson_realloc(void* p, size_t size) { return realloc(p, size); } 56 | inline void pjson_free(void* p) { free(p); } 57 | 58 | // Misc. Helpers 59 | template inline void swap(T& l, T& r) { T temp(l); l = r; r = temp; } 60 | 61 | inline bool is_power_of_2(uint32 x) { return x && ((x & (x - 1U)) == 0U); } 62 | inline uint32 next_pow2(uint32 val) { val--; val |= val >> 16; val |= val >> 8; val |= val >> 4; val |= val >> 2; val |= val >> 1; return val + 1; } 63 | 64 | #ifdef _MSC_VER 65 | inline int pjson_stricmp(const char* p, const char* q) { return _stricmp(p, q); } 66 | #else 67 | inline int pjson_stricmp(const char* p, const char* q) { return strcasecmp(p, q); } 68 | #endif 69 | 70 | // ---- Global Arrays 71 | 72 | // This template utilizes the One Definition Rule to create global arrays in a header. 73 | template 74 | struct globals_struct 75 | { 76 | static const uint8 s_str_serialize_flags[256]; 77 | static const double s_pow10_table[63]; 78 | static const uint8 s_parse_flags[256]; 79 | }; 80 | typedef globals_struct<> globals; 81 | 82 | template 83 | const uint8 globals_struct::s_str_serialize_flags[256] = 84 | { 85 | // 0 1 2 3 4 5 6 7 8 9 A B C D E F 86 | 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 0 87 | 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 1 88 | 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 2 89 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1 90 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 4 91 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, // 5 92 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 6 93 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 7 94 | // 128-255 95 | 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 96 | 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 97 | 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 98 | 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0 99 | }; 100 | 101 | template 102 | const double globals_struct::s_pow10_table[63] = 103 | { 104 | 1.e-031,1.e-030,1.e-029,1.e-028,1.e-027,1.e-026,1.e-025,1.e-024,1.e-023,1.e-022,1.e-021,1.e-020,1.e-019,1.e-018,1.e-017,1.e-016, 105 | 1.e-015,1.e-014,1.e-013,1.e-012,1.e-011,1.e-010,1.e-009,1.e-008,1.e-007,1.e-006,1.e-005,1.e-004,1.e-003,1.e-002,1.e-001,1.e+000, 106 | 1.e+001,1.e+002,1.e+003,1.e+004,1.e+005,1.e+006,1.e+007,1.e+008,1.e+009,1.e+010,1.e+011,1.e+012,1.e+013,1.e+014,1.e+015,1.e+016, 107 | 1.e+017,1.e+018,1.e+019,1.e+020,1.e+021,1.e+022,1.e+023,1.e+024,1.e+025,1.e+026,1.e+027,1.e+028,1.e+029,1.e+030,1.e+031 108 | }; 109 | 110 | // bit 0 (1) - set if: \0 cr lf " \ 111 | // bit 1 (2) - set if: \0 cr lf 112 | // bit 2 (4) - set if: whitespace 113 | // bit 3 (8) - set if: 0-9 114 | // bit 4 (0x10) - set if: 0-9 e E . 115 | template 116 | const uint8 globals_struct::s_parse_flags[256] = 117 | { 118 | // 0 1 2 3 4 5 6 7 8 9 A B C D E F 119 | 7, 4, 4, 4, 4, 4, 4, 4, 4, 4, 7, 4, 4, 7, 4, 4, // 0 120 | 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, // 1 121 | 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x10,0, // 2 122 | 0x18,0x18,0x18,0x18,0x18,0x18,0x18,0x18, 0x18,0x18,0, 0, 0, 0, 0, 0, // 3 123 | 0, 0, 0, 0, 0, 0x10,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 4 124 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, // 5 125 | 0, 0, 0, 0, 0, 0x10,0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 6 126 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 7 127 | 128 | // 128-255 129 | 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 130 | 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 131 | 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 132 | 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0 133 | }; 134 | 135 | // ---- Pool Allocator 136 | 137 | struct pool_allocator 138 | { 139 | inline pool_allocator(uint initial_size = 0, uint min_chunk_size = PJSON_DEFAULT_MIN_CHUNK_SIZE, size_t max_bytes_to_preserve_across_resets = PJSON_DEFAULT_MAX_BYTES_TO_PRESERVE_ACROSS_RESETS) : 140 | m_pActive_chunks(NULL), 141 | m_pFree_chunks(NULL), 142 | m_total_free_bytes(0), 143 | m_initial_size(initial_size), 144 | m_min_chunk_size(min_chunk_size), 145 | m_max_to_preserve_across_resets(max_bytes_to_preserve_across_resets), 146 | m_cur_grow_size(min_chunk_size) 147 | { 148 | if (initial_size) 149 | { 150 | m_pActive_chunks = static_cast(pjson_malloc(sizeof(chunk) + initial_size)); 151 | m_pActive_chunks->m_pNext = NULL; 152 | m_pActive_chunks->m_ofs = 0; 153 | m_pActive_chunks->m_size = initial_size; 154 | } 155 | } 156 | 157 | inline ~pool_allocator() 158 | { 159 | clear(); 160 | } 161 | 162 | // Release all active/free chunks 163 | void clear() 164 | { 165 | free_chunk_chain(m_pActive_chunks); 166 | m_pActive_chunks = NULL; 167 | 168 | free_chunk_chain(m_pFree_chunks); 169 | m_pFree_chunks = NULL; 170 | 171 | m_total_free_bytes = 0; 172 | 173 | m_cur_grow_size = m_min_chunk_size; 174 | } 175 | 176 | inline size_t get_total_free_bytes() const { return m_total_free_bytes; } 177 | 178 | inline uint get_min_chunk_size() const { return m_min_chunk_size; } 179 | inline size_t get_max_bytes_to_preserve_across_resets() const { return m_max_to_preserve_across_resets; } 180 | 181 | inline void set_min_chunk_size(uint s) { m_min_chunk_size = m_cur_grow_size = s; } 182 | inline void set_max_bytes_to_preserve_across_resets(size_t s) { m_max_to_preserve_across_resets = s; } 183 | 184 | inline uint get_cur_grow_size() const { return m_cur_grow_size; } 185 | 186 | inline void* Alloc(size_t size) 187 | { 188 | size = (size + 3) & ~3; 189 | if ((!m_pActive_chunks) || ((m_pActive_chunks->m_size - m_pActive_chunks->m_ofs) < size)) 190 | { 191 | chunk* pNew_chunk = m_pFree_chunks; 192 | if ((pNew_chunk) && (pNew_chunk->m_size >= size)) 193 | { 194 | PJSON_ASSERT(m_total_free_bytes >= pNew_chunk->m_size); 195 | m_total_free_bytes -= pNew_chunk->m_size; 196 | m_pFree_chunks = pNew_chunk->m_pNext; 197 | PJSON_ASSERT(!pNew_chunk->m_ofs); 198 | } 199 | else 200 | { 201 | size_t alloc_size = PJSON_MAX(size, m_cur_grow_size); 202 | m_cur_grow_size = PJSON_MIN(m_cur_grow_size * 2, PJSON_MAX_CHUNK_GROW_SIZE); 203 | 204 | pNew_chunk = static_cast(pjson_malloc(sizeof(chunk) + alloc_size)); 205 | pNew_chunk->m_size = alloc_size; 206 | pNew_chunk->m_ofs = 0; 207 | } 208 | 209 | pNew_chunk->m_pNext = m_pActive_chunks; 210 | m_pActive_chunks = pNew_chunk; 211 | } 212 | void* pRet = (uint8*)m_pActive_chunks + sizeof(chunk) + m_pActive_chunks->m_ofs; 213 | m_pActive_chunks->m_ofs += size; 214 | PJSON_ASSERT(m_pActive_chunks->m_ofs <= m_pActive_chunks->m_size); 215 | return pRet; 216 | } 217 | 218 | inline void* Realloc(void* p, size_t new_size, size_t cur_size) 219 | { 220 | if (!p) 221 | return Alloc(new_size); 222 | 223 | new_size = (new_size + 3) & ~3; 224 | cur_size = (cur_size + 3) & ~3; 225 | if (new_size == cur_size) 226 | return p; 227 | 228 | uint8* pTop = (uint8*)m_pActive_chunks + sizeof(chunk) + m_pActive_chunks->m_ofs; 229 | if ((static_cast(p) + cur_size) == pTop) 230 | { 231 | if (new_size > cur_size) 232 | { 233 | size_t bytes_needed = new_size - cur_size; 234 | if ((m_pActive_chunks->m_size - m_pActive_chunks->m_ofs) >= bytes_needed) 235 | { 236 | m_pActive_chunks->m_ofs += bytes_needed; 237 | PJSON_ASSERT(m_pActive_chunks->m_ofs <= m_pActive_chunks->m_size); 238 | return p; 239 | } 240 | } 241 | else 242 | { 243 | PJSON_ASSERT(m_pActive_chunks->m_ofs >= (cur_size - new_size)); 244 | m_pActive_chunks->m_ofs -= (cur_size - new_size); 245 | return new_size ? p : NULL; 246 | } 247 | } 248 | 249 | if (!new_size) 250 | return NULL; 251 | 252 | void* pNew_block = Alloc(new_size); 253 | memcpy(pNew_block, p, cur_size); 254 | return pNew_block; 255 | } 256 | 257 | // Move all active chunks to the free chunk list, then free any chunks if we're over the preserve limit. 258 | inline void reset() 259 | { 260 | if (!m_pActive_chunks) 261 | return; 262 | 263 | chunk* pCur_active_tail = m_pActive_chunks; 264 | size_t total_allocated_bytes = 0; 265 | for ( ; ; ) 266 | { 267 | total_allocated_bytes += pCur_active_tail->m_size; 268 | pCur_active_tail->m_ofs = 0; 269 | if (!pCur_active_tail->m_pNext) 270 | break; 271 | pCur_active_tail = pCur_active_tail->m_pNext; 272 | } 273 | pCur_active_tail->m_pNext = m_pFree_chunks; 274 | 275 | m_pFree_chunks = m_pActive_chunks; 276 | m_pActive_chunks = NULL; 277 | 278 | m_total_free_bytes += total_allocated_bytes; 279 | while (m_total_free_bytes > m_max_to_preserve_across_resets) 280 | { 281 | PJSON_ASSERT(m_pFree_chunks); 282 | chunk* pNext_free = m_pFree_chunks->m_pNext; 283 | PJSON_ASSERT(m_total_free_bytes >= m_pFree_chunks->m_size); 284 | m_total_free_bytes -= m_pFree_chunks->m_size; 285 | pjson_free(m_pFree_chunks); 286 | m_pFree_chunks = pNext_free; 287 | } 288 | 289 | m_cur_grow_size = m_min_chunk_size; 290 | } 291 | 292 | struct stats_t 293 | { 294 | size_t m_total_allocated; 295 | 296 | uint m_num_active_chunks; 297 | size_t m_num_active_bytes_allocated; 298 | size_t m_num_active_bytes_avail; 299 | size_t m_max_active_chunk_size; 300 | 301 | uint m_num_free_chunks; 302 | size_t m_num_free_chunk_bytes_avail; 303 | size_t m_max_free_chunk_size; 304 | }; 305 | 306 | inline void get_stats(stats_t& s) const 307 | { 308 | memset(&s, 0, sizeof(s)); 309 | 310 | chunk* pChunk = m_pActive_chunks; 311 | while (pChunk) 312 | { 313 | s.m_num_active_chunks++; 314 | s.m_total_allocated += pChunk->m_size; 315 | s.m_num_active_bytes_allocated += pChunk->m_ofs; 316 | s.m_num_active_bytes_avail += (pChunk->m_size - pChunk->m_ofs); 317 | s.m_max_active_chunk_size = PJSON_MAX(s.m_max_active_chunk_size, pChunk->m_size); 318 | pChunk = pChunk->m_pNext; 319 | } 320 | 321 | pChunk = m_pFree_chunks; 322 | while (pChunk) 323 | { 324 | s.m_num_free_chunks++; 325 | s.m_total_allocated += pChunk->m_size; 326 | PJSON_ASSERT(!pChunk->m_ofs); 327 | s.m_num_free_chunk_bytes_avail += pChunk->m_size; 328 | s.m_max_free_chunk_size = PJSON_MAX(s.m_max_free_chunk_size, pChunk->m_size); 329 | pChunk = pChunk->m_pNext; 330 | } 331 | 332 | PJSON_ASSERT(s.m_num_free_chunk_bytes_avail == m_total_free_bytes); 333 | } 334 | 335 | private: 336 | pool_allocator(const pool_allocator&); 337 | pool_allocator& operator= (const pool_allocator&); 338 | 339 | struct chunk 340 | { 341 | chunk* m_pNext; 342 | size_t m_size; 343 | size_t m_ofs; 344 | }; 345 | 346 | chunk* m_pActive_chunks; 347 | chunk* m_pFree_chunks; 348 | size_t m_total_free_bytes; 349 | 350 | uint m_initial_size; 351 | uint m_min_chunk_size; 352 | size_t m_max_to_preserve_across_resets; 353 | 354 | uint m_cur_grow_size; 355 | 356 | inline void free_chunk_chain(chunk* pChunk) 357 | { 358 | while (pChunk) 359 | { 360 | chunk* pNext_chunk = pChunk->m_pNext; 361 | pjson_free(pChunk); 362 | pChunk = pNext_chunk; 363 | } 364 | } 365 | }; 366 | 367 | // ---- Simple vector (growable array) 368 | 369 | template 370 | struct simple_vector_default_copy_construction_policy 371 | { 372 | inline static void copy_construct(void *pDst, const T& init, pool_allocator& alloc) { (void)alloc; new (pDst) T(init); } 373 | inline static void assign(void *pDst, const T& src, pool_allocator& alloc) { (void)alloc; *static_cast(pDst) = src; } 374 | }; 375 | 376 | template 377 | struct simple_vector_allocator_copy_construction_policy 378 | { 379 | inline static void copy_construct(void *pDst, const T& init, pool_allocator& alloc) { (void)alloc; new (pDst) T(init, alloc); } 380 | inline static void assign(void *pDst, const T& src, pool_allocator& alloc) { static_cast(pDst)->assign(src, alloc); } 381 | }; 382 | 383 | template inline T* construct(T* p) { return new (static_cast(p)) T; } 384 | template inline void construct_array(T* p, uint n) { T* q = p + n; for ( ; p != q; ++p) new (static_cast(p)) T; } 385 | 386 | template 387 | struct elemental_vector 388 | { 389 | typedef T value_type; 390 | typedef T& reference; 391 | typedef const T& const_reference; 392 | typedef T* pointer; 393 | typedef const T* const_pointer; 394 | 395 | T* m_p; 396 | uint32 m_size; 397 | }; 398 | 399 | template > 400 | struct simple_vector : elemental_vector 401 | { 402 | inline simple_vector() { construct(); } 403 | inline simple_vector(const simple_vector& other, pool_allocator& alloc) { construct(other, alloc); } 404 | 405 | // Manual constructor methods 406 | inline void construct() { this->m_p = NULL; this->m_size = 0; } 407 | inline void construct(uint size, pool_allocator& alloc) { construct(); enlarge(size, alloc, false); } 408 | inline void construct(const T* p, uint size, pool_allocator& alloc) 409 | { 410 | this->m_size = size; 411 | this->m_p = NULL; 412 | if (size) 413 | { 414 | uint num_bytes = sizeof(T) * size; 415 | this->m_p = static_cast(alloc.Alloc(num_bytes)); 416 | if (UseConstructor) 417 | { 418 | T* pDst = this->m_p; 419 | T* pDst_end = pDst + size; 420 | const T* pSrc = p; 421 | while (pDst != pDst_end) 422 | ConstructionPolicy::copy_construct(pDst++, *pSrc++, alloc); 423 | } 424 | else 425 | memcpy(this->m_p, p, num_bytes); 426 | } 427 | } 428 | inline void construct(const simple_vector& other, pool_allocator& alloc) 429 | { 430 | construct(other.m_p, other.m_size, alloc); 431 | } 432 | 433 | inline uint size() const { return this->m_size; } 434 | inline uint size_in_bytes() const { return this->m_size * sizeof(T); } 435 | 436 | inline const T& operator[] (uint i) const { PJSON_ASSERT(i < this->m_size); return this->m_p[i]; } 437 | inline T& operator[] (uint i) { PJSON_ASSERT(i < this->m_size); return this->m_p[i]; } 438 | 439 | inline const T* get_ptr() const { return this->m_p; } 440 | inline T* get_ptr() { return this->m_p; } 441 | 442 | inline const T* get_ptr(const T* pDef) const { return this->m_p ? this->m_p : pDef; } 443 | inline T* get_ptr(T* pDef) { return this->m_p ? this->m_p : pDef; } 444 | 445 | inline void clear() { this->m_p = NULL; this->m_size = 0; } 446 | 447 | inline void resize(uint new_size, pool_allocator& alloc) 448 | { 449 | if (new_size > this->m_size) 450 | { 451 | grow(new_size, alloc); 452 | 453 | if (UseConstructor) 454 | construct_array(this->m_p + this->m_size, new_size - this->m_size); 455 | } 456 | 457 | this->m_size = new_size; 458 | } 459 | 460 | inline void shrink(uint new_size) 461 | { 462 | this->m_size = new_size; 463 | } 464 | 465 | inline T* enlarge_no_construct(uint n, pool_allocator& alloc) 466 | { 467 | PJSON_ASSERT(n); 468 | uint cur_size = this->m_size, new_size = this->m_size + n; 469 | grow(new_size, alloc); 470 | this->m_size = new_size; 471 | return this->m_p + cur_size; 472 | } 473 | 474 | inline T* enlarge(uint n, pool_allocator& alloc) 475 | { 476 | T* p = enlarge_no_construct(n, alloc); 477 | if (UseConstructor) 478 | construct_array(p, n); 479 | return p; 480 | } 481 | 482 | inline void push_back(const T& obj, pool_allocator& alloc) 483 | { 484 | PJSON_ASSERT(!this->m_p || (&obj < this->m_p) || (&obj >= (this->m_p + this->m_size))); 485 | grow(this->m_size + 1, alloc); 486 | if (UseConstructor) 487 | ConstructionPolicy::copy_construct(this->m_p + this->m_size, obj, alloc); 488 | else 489 | memcpy(this->m_p + this->m_size, &obj, sizeof(T)); 490 | this->m_size++; 491 | } 492 | 493 | inline void push_back(const T* p, uint n, pool_allocator& alloc) 494 | { 495 | PJSON_ASSERT(!this->m_p || ((p + n) <= this->m_p) || (p >= (this->m_p + this->m_size))); 496 | T* pDst = enlarge_no_construct(n, alloc); 497 | if (UseConstructor) 498 | { 499 | T* pDst_end = pDst + n; 500 | const T* pSrc = p; 501 | while (pDst != pDst_end) 502 | ConstructionPolicy::copy_construct(pDst, *pSrc++, alloc); 503 | } 504 | else 505 | memcpy(pDst, p, sizeof(T) * n); 506 | } 507 | 508 | inline void assign(const T* p, uint n, pool_allocator& alloc) 509 | { 510 | PJSON_ASSERT(!this->m_p || ((p + n) <= this->m_p) || (p >= (this->m_p + this->m_size))); 511 | 512 | const uint num_to_assign = PJSON_MIN(this->m_size, n); 513 | if (num_to_assign) 514 | { 515 | if (UseConstructor) 516 | { 517 | for (uint i = 0; i < num_to_assign; ++i) 518 | ConstructionPolicy::assign(&this->m_p[i], p[i], alloc); 519 | } 520 | else 521 | memcpy(this->m_p, p, sizeof(T) * num_to_assign); 522 | } 523 | 524 | if (n > this->m_size) 525 | push_back(p + num_to_assign, n - num_to_assign, alloc); 526 | else 527 | shrink(n); 528 | } 529 | 530 | inline void assign(const simple_vector& other, pool_allocator& alloc) 531 | { 532 | assign(other.m_p, other.m_size, alloc); 533 | } 534 | 535 | inline void erase(uint start, uint n) 536 | { 537 | PJSON_ASSERT((start + n) <= this->m_size); 538 | if ((!n) || ((start + n) > this->m_size)) 539 | return; 540 | 541 | const uint num_to_move = this->m_size - (start + n); 542 | 543 | T* pDst = this->m_p + start; 544 | 545 | memmove(pDst, this->m_p + start + n, num_to_move * sizeof(T)); 546 | 547 | this->m_size -= n; 548 | } 549 | 550 | inline void swap(simple_vector& other) 551 | { 552 | pjson::swap(this->m_p, other.m_p); 553 | pjson::swap(this->m_size, other.m_size); 554 | } 555 | 556 | inline void grow(uint new_size, pool_allocator& alloc) 557 | { 558 | if (new_size > this->m_size) 559 | this->m_p = static_cast(alloc.Realloc(this->m_p, sizeof(T) * new_size, this->m_size * sizeof(T))); 560 | } 561 | }; 562 | 563 | enum json_value_type_t 564 | { 565 | cJSONValueTypeNull = 0, 566 | cJSONValueTypeBool, 567 | cJSONValueTypeInt, 568 | cJSONValueTypeDouble, 569 | 570 | // All types that follow require storage. Do not change the relative order of these types. 571 | cJSONValueTypeString, 572 | cJSONValueTypeArray, 573 | cJSONValueTypeObject, 574 | }; 575 | 576 | // ---- struct value_variant_data 577 | 578 | typedef simple_vector string_vec_t; 579 | typedef simple_vector > key_value_vec_t; 580 | typedef simple_vector > value_variant_vec_t; 581 | 582 | #pragma pack(push, 4) 583 | struct value_variant_data 584 | { 585 | union json_value_data_t 586 | { 587 | elemental_vector m_object; 588 | elemental_vector m_array; 589 | elemental_vector m_string; 590 | int64 m_nVal; 591 | double m_flVal; 592 | }; 593 | 594 | json_value_data_t m_data; 595 | json_value_type_t m_type; 596 | 597 | inline const string_vec_t& get_string() const { return (const string_vec_t&)m_data.m_string; } 598 | inline string_vec_t& get_string() { return (string_vec_t&)m_data.m_string; } 599 | inline const char* get_string_ptr() const { return get_string().get_ptr(""); } 600 | 601 | inline const value_variant_vec_t& get_array() const { return (const value_variant_vec_t&)m_data.m_array; } 602 | inline value_variant_vec_t& get_array() { return (value_variant_vec_t&)m_data.m_array; } 603 | 604 | inline const key_value_vec_t& get_object() const { return (const key_value_vec_t&)m_data.m_object; } 605 | inline key_value_vec_t& get_object() { return (key_value_vec_t&)m_data.m_object; } 606 | }; 607 | 608 | // ---- struct key_value_t 609 | 610 | struct key_value_t 611 | { 612 | inline key_value_t() { } 613 | inline key_value_t(const key_value_t& other, pool_allocator& alloc); 614 | 615 | inline void assign(const key_value_t& src, pool_allocator& alloc); 616 | 617 | inline const string_vec_t& get_key() const { return m_key; } 618 | inline string_vec_t& get_key() { return m_key; } 619 | 620 | inline const value_variant& get_value() const { return (const value_variant&)m_value_data; } 621 | inline value_variant& get_value() { return (value_variant&)m_value_data; } 622 | 623 | string_vec_t m_key; 624 | value_variant_data m_value_data; 625 | }; 626 | #pragma pack(pop) 627 | 628 | // ---- class char_vector_print_helper 629 | 630 | class char_vector_print_helper 631 | { 632 | char_vector_print_helper(const char_vector_print_helper&); 633 | char_vector_print_helper& operator= (const char_vector_print_helper&); 634 | 635 | public: 636 | inline char_vector_print_helper(char_vec_t& buf) : m_buf(buf) { } 637 | 638 | inline void resize(size_t new_size) { m_buf.resize(new_size); } 639 | inline size_t size() const { return m_buf.size(); } 640 | inline char* get_ptr() const { return &m_buf[0]; } 641 | 642 | inline const char_vec_t& get_buf() const { return m_buf; } 643 | inline char_vec_t& get_buf() { return m_buf; } 644 | 645 | inline void puts(const char* pStr, size_t l) { m_buf.insert(m_buf.end(), pStr, pStr + l); } 646 | inline void print_tabs(size_t n) { m_buf.insert(m_buf.end(), n, '\t'); } 647 | inline void print_char(char c) { m_buf.push_back(c); } 648 | 649 | void print_escaped(const string_vec_t& str) 650 | { 651 | const char* pStr = str.m_p; 652 | uint len = str.m_size; (void)len; 653 | 654 | static const char* s_to_hex = "0123456789abcdef"; 655 | print_char('\"'); 656 | while (*pStr) 657 | { 658 | uint8 c = *pStr++; 659 | if ((c >= ' ') && (c != '\"') && (c != '\\')) 660 | print_char(c); 661 | else 662 | { 663 | print_char('\\'); 664 | switch (c) 665 | { 666 | case '\b': print_char('b'); break; 667 | case '\r': print_char('r'); break; 668 | case '\t': print_char('t'); break; 669 | case '\f': print_char('f'); break; 670 | case '\n': print_char('n'); break; 671 | case '\\': print_char('\\'); break; 672 | case '\"': print_char('\"'); break; 673 | default: puts("u00", 3); print_char(s_to_hex[c >> 4]); print_char(s_to_hex[c & 0xF]); break; 674 | } 675 | } 676 | } 677 | print_char('\"'); 678 | } 679 | 680 | private: 681 | char_vec_t& m_buf; 682 | }; 683 | 684 | // ---- class char_buf_print_helper 685 | 686 | class char_buf_print_helper 687 | { 688 | char_buf_print_helper(const char_buf_print_helper&); 689 | char_buf_print_helper& operator= (const char_buf_print_helper&); 690 | 691 | public: 692 | inline char_buf_print_helper(char* pBuf, size_t buf_size) : m_pDst(pBuf), m_pStart(pBuf), m_pEnd(pBuf + buf_size) { } 693 | 694 | inline void resize(size_t new_size) { PJSON_ASSERT(new_size <= (size_t)(m_pEnd - m_pStart)); m_pDst = m_pStart + new_size; } 695 | inline size_t size() const { return m_pDst - m_pStart; } 696 | inline char* get_ptr() const { return m_pStart; } 697 | 698 | inline void puts(const char* pStr, size_t l) { memcpy(m_pDst, pStr, l = PJSON_MIN(l, (size_t)(m_pEnd - m_pDst))); m_pDst += l; } 699 | inline void print_tabs(size_t n) { n = PJSON_MIN(n, (size_t)(m_pEnd - m_pDst)); memset(m_pDst, '\t', n); m_pDst += n; } 700 | inline void print_char(char c) { if (m_pDst < m_pEnd) *m_pDst++ = c; } 701 | 702 | void print_escaped(const string_vec_t& str) 703 | { 704 | static const char* s_to_hex = "0123456789abcdef"; 705 | 706 | const char* pStr = str.m_p; 707 | char* pDst = m_pDst; 708 | char* pEnd = m_pEnd; 709 | uint len = str.m_size; 710 | 711 | // If len!=0, it includes the terminating null, so this expression is conservative. 712 | if (static_cast(pEnd - pDst) < (len + 2)) { m_pDst = pEnd; return; } 713 | 714 | *pDst++ = '\"'; 715 | 716 | uint8 c = 0; if (pStr) c = pStr[0]; 717 | while (!globals::s_str_serialize_flags[c]) 718 | { 719 | pDst[0] = c; c = pStr[1]; if (globals::s_str_serialize_flags[c]) { ++pStr, ++pDst; break; } 720 | pDst[1] = c; c = pStr[2]; if (globals::s_str_serialize_flags[c]) { pStr += 2, pDst += 2; break; } 721 | pDst[2] = c; c = pStr[3]; if (globals::s_str_serialize_flags[c]) { pStr += 3, pDst += 3; break; } 722 | pDst[3] = c; c = pStr[4]; pStr += 4, pDst += 4; 723 | } 724 | 725 | while (c) 726 | { 727 | if ((pEnd - pDst) < 7) 728 | { 729 | m_pDst = pEnd; 730 | return; 731 | } 732 | if (!globals::s_str_serialize_flags[c]) 733 | *pDst++ = c; 734 | else 735 | { 736 | pDst[0] = '\\'; 737 | switch (c) 738 | { 739 | case '\b': pDst[1] = 'b'; break; 740 | case '\r': pDst[1] = 'r'; break; 741 | case '\t': pDst[1] = 't'; break; 742 | case '\f': pDst[1] = 'f'; break; 743 | case '\n': pDst[1] = 'n'; break; 744 | case '\\': pDst[1] = '\\'; break; 745 | case '\"': pDst[1] = '\"'; break; 746 | default: pDst[1] = 'u', pDst[2] = '0', pDst[3] = '0', pDst[4] = s_to_hex[c >> 4], pDst[5] = s_to_hex[c & 0xF]; pDst += 3; break; 747 | } 748 | pDst += 2; 749 | } 750 | c = *pStr++; 751 | } 752 | 753 | *pDst++ = '\"'; 754 | PJSON_ASSERT(pDst <= pEnd); 755 | m_pDst = pDst; 756 | } 757 | 758 | private: 759 | char* m_pDst, *m_pStart, *m_pEnd; 760 | }; 761 | 762 | // ---- class serialize_helper 763 | 764 | template 765 | class serialize_helper : public T 766 | { 767 | public: 768 | typedef T base; 769 | 770 | template inline serialize_helper(I& init) : T(init) { } 771 | template inline serialize_helper(I& init1, J& init2) : T(init1, init2) { } 772 | 773 | inline void puts(const char* pStr) { T::puts(pStr, strlen(pStr)); } 774 | inline void puts(const char* pStr, size_t l) { T::puts(pStr, l); } 775 | }; 776 | 777 | // ---- class value_variant 778 | 779 | #pragma pack(push, 4) 780 | class value_variant : public value_variant_data 781 | { 782 | friend document; 783 | friend key_value_t; 784 | 785 | value_variant(const value_variant&); 786 | value_variant& operator= (const value_variant&); 787 | 788 | public: 789 | inline value_variant() { m_type = cJSONValueTypeNull; m_data.m_nVal = 0; } 790 | inline value_variant(bool val) { m_type = cJSONValueTypeBool; m_data.m_nVal = val; } 791 | inline value_variant(int32 nVal) { m_type = cJSONValueTypeInt; m_data.m_nVal = nVal; } 792 | inline value_variant(uint32 nVal) { m_type = cJSONValueTypeInt; m_data.m_nVal = nVal; } 793 | inline value_variant(int64 nVal) { m_type = cJSONValueTypeInt; m_data.m_nVal = nVal; } 794 | inline value_variant(double flVal) { m_type = cJSONValueTypeDouble; m_data.m_flVal = flVal; } 795 | 796 | inline value_variant(const char* pStr, pool_allocator& alloc) 797 | { 798 | m_type = cJSONValueTypeString; 799 | if (!pStr) pStr = ""; 800 | get_string().construct(pStr, static_cast(strlen(pStr)) + 1, alloc); 801 | } 802 | 803 | inline value_variant(json_value_type_t type) 804 | { 805 | construct(type); 806 | } 807 | 808 | inline value_variant(const value_variant& other, pool_allocator& alloc) 809 | { 810 | construct(other, alloc); 811 | } 812 | 813 | inline value_variant &assign(const value_variant& rhs, pool_allocator& alloc) 814 | { 815 | if (this == &rhs) 816 | return *this; 817 | if ((m_type >= cJSONValueTypeString) && (m_type == rhs.m_type)) 818 | { 819 | if (is_string()) 820 | get_string().assign(rhs.get_string(), alloc); 821 | else if (is_object()) 822 | get_object().assign(rhs.get_object(), alloc); 823 | else 824 | get_array().assign(rhs.get_array(), alloc); 825 | } 826 | else 827 | { 828 | construct(rhs, alloc); 829 | } 830 | return *this; 831 | } 832 | 833 | inline json_value_type_t get_type() const { return m_type; } 834 | 835 | inline bool is_null() const { return m_type == cJSONValueTypeNull; } 836 | inline bool is_valid() const { return m_type != cJSONValueTypeNull; } 837 | inline bool is_bool() const { return m_type == cJSONValueTypeBool; } 838 | inline bool is_int() const { return m_type == cJSONValueTypeInt; } 839 | inline bool is_double() const { return m_type == cJSONValueTypeDouble; } 840 | inline bool is_numeric() const { return (m_type == cJSONValueTypeInt) || (m_type == cJSONValueTypeDouble); } 841 | inline bool is_string() const { return m_type == cJSONValueTypeString; } 842 | inline bool is_object_or_array() const { return m_type >= cJSONValueTypeArray; } 843 | inline bool is_object() const { return m_type == cJSONValueTypeObject; } 844 | inline bool is_array() const { return m_type == cJSONValueTypeArray; } 845 | 846 | inline void clear() { set_to_null(); } 847 | 848 | inline void assume_ownership(value_variant& src_val) { set_to_null(); swap(src_val); } 849 | inline void release_ownership(value_variant& dst_value) { dst_value.set_to_null(); dst_value.swap(*this); } 850 | 851 | inline value_variant& set_to_object() { construct(cJSONValueTypeObject); return *this; } 852 | inline value_variant& set_to_array() { construct(cJSONValueTypeArray); return *this; } 853 | inline value_variant& set_to_node(bool is_object) { construct(is_object ? cJSONValueTypeObject : cJSONValueTypeArray); return *this; } 854 | 855 | inline value_variant& set_to_null() { m_data.m_nVal = 0; m_type = cJSONValueTypeNull; return *this; } 856 | inline value_variant& set(bool val) { m_data.m_nVal = val; m_type = cJSONValueTypeBool; return *this; } 857 | inline value_variant& set(int32 nVal) { m_data.m_nVal = nVal; m_type = cJSONValueTypeInt; return *this; } 858 | inline value_variant& set(int64 nVal) { m_data.m_nVal = nVal; m_type = cJSONValueTypeInt; return *this; } 859 | inline value_variant& set(uint32 nVal) { set(static_cast(nVal)); return *this; } 860 | inline value_variant& set(double flVal) { m_data.m_flVal = flVal; m_type = cJSONValueTypeDouble; return *this; } 861 | 862 | inline value_variant& set(const char* pStr, pool_allocator& alloc) 863 | { 864 | if (!pStr) pStr = ""; 865 | uint l = static_cast(strlen(pStr)) + 1; 866 | if (!is_string()) 867 | { 868 | m_type = cJSONValueTypeString; 869 | get_string().construct(pStr, l, alloc); 870 | } 871 | else 872 | get_string().assign(pStr, l, alloc); 873 | return *this; 874 | } 875 | 876 | inline value_variant& set_assume_ownership(char* pStr, uint len) 877 | { 878 | m_type = cJSONValueTypeString; 879 | string_vec_t& str = get_string(); 880 | str.m_p = pStr; 881 | str.m_size = len; 882 | return *this; 883 | } 884 | 885 | inline value_variant& set(const value_variant* pVals, uint n, pool_allocator& alloc) 886 | { 887 | if (!is_array()) 888 | { 889 | m_type = cJSONValueTypeArray; 890 | get_array().construct(pVals, n, alloc); 891 | } 892 | else 893 | get_array().assign(pVals, n, alloc); 894 | return *this; 895 | } 896 | 897 | inline value_variant& set_assume_ownership(value_variant* pVals, uint n) 898 | { 899 | m_type = cJSONValueTypeArray; 900 | value_variant_vec_t& arr = get_array(); 901 | arr.m_p = pVals; 902 | arr.m_size = n; 903 | return *this; 904 | } 905 | 906 | inline value_variant& set(const key_value_t* pKey_values, uint n, pool_allocator& alloc) 907 | { 908 | if (!is_object()) 909 | { 910 | m_type = cJSONValueTypeObject; 911 | get_object().construct(pKey_values, n, alloc); 912 | } 913 | else 914 | get_object().assign(pKey_values, n, alloc); 915 | return *this; 916 | } 917 | 918 | inline value_variant& set_assume_ownership(key_value_t* pKey_values, uint n) 919 | { 920 | m_type = cJSONValueTypeObject; 921 | key_value_vec_t& obj = get_object(); 922 | obj.m_p = pKey_values; 923 | obj.m_size = n; 924 | return *this; 925 | } 926 | 927 | inline value_variant &operator=(bool val) { set(val); return *this; } 928 | inline value_variant &operator=(int32 nVal) { set(nVal); return *this; } 929 | inline value_variant &operator=(uint32 nVal) { set(nVal); return *this; } 930 | inline value_variant &operator=(int64 nVal) { set(nVal); return *this; } 931 | inline value_variant &operator=(double flVal) { set(flVal); return *this; } 932 | 933 | inline bool get_bool_value(bool& val, bool def = false) const { if (is_bool()) { val = (m_data.m_nVal != 0); return true; } else return convert_to_bool(val, def); } 934 | inline bool get_numeric_value(int32& val, int32 def = 0) const { if ((is_int()) && (m_data.m_nVal == static_cast(m_data.m_nVal))) { val = static_cast(m_data.m_nVal); return true; } else return convert_to_int32(val, def); } 935 | inline bool get_numeric_value(int64& val, int64 def = 0) const { if (is_int()) { val = m_data.m_nVal; return true; } else return convert_to_int64(val, def); } 936 | inline bool get_numeric_value(float& val, float def = 0.0f) const { if (is_double()) { val = static_cast(m_data.m_flVal); return true; } else return convert_to_float(val, def); } 937 | inline bool get_numeric_value(double& val, double def = 0.0f) const { if (is_double()) { val = m_data.m_flVal; return true; } else return convert_to_double(val, def); } 938 | inline bool get_string_value(string_t& val, const char* pDef = "") const { if (is_string()) { val = get_string_ptr(); return true; } else return convert_to_string(val, pDef); } 939 | 940 | inline bool as_bool(bool def = false) const { bool result; get_bool_value(result, def); return result; } 941 | inline int as_int32(int32 def = 0) const { int32 result; get_numeric_value(result, def); return result; } 942 | inline int64 as_int64(int64 def = 0) const { int64 result; get_numeric_value(result, def); return result; } 943 | inline float as_float(float def = 0.0f) const { float result; get_numeric_value(result, def); return result; } 944 | inline double as_double(double def = 0.0f) const { double result; get_numeric_value(result, def); return result; } 945 | 946 | // Returns value as a string, or the default string if the value cannot be converted. 947 | inline string_t as_string(const char* pDef = "") const { string_t result; get_string_value(result, pDef); return result; } 948 | 949 | // Returns pointer to null terminated string or NULL if the value is not a string. 950 | inline const char* as_string_ptr() const { return is_string() ? get_string_ptr() : NULL; } 951 | 952 | inline void swap(value_variant& other) 953 | { 954 | pjson::swap(m_type, other.m_type); 955 | get_object().swap(other.get_object()); 956 | } 957 | 958 | inline uint size() const { PJSON_ASSERT(is_object_or_array()); return is_object_or_array() ? get_array().size() : 0; } 959 | 960 | inline const char *get_key_name_at_index(uint index) const { PJSON_ASSERT(is_object()); return get_object()[index].get_key().get_ptr(""); } 961 | 962 | inline const value_variant *find_child_array(const char *pName) const 963 | { 964 | int index = find_key(pName); 965 | if ((index >= 0) && (get_object()[index].get_value().is_array())) 966 | return &get_object()[index].get_value(); 967 | return NULL; 968 | } 969 | 970 | inline const value_variant *find_child_object(const char *pName) const 971 | { 972 | int index = find_key(pName); 973 | if ((index >= 0) && (get_object()[index].get_value().is_object())) 974 | return &get_object()[index].get_value(); 975 | return NULL; 976 | } 977 | 978 | inline const value_variant *find_value_variant(const char *pName) const 979 | { 980 | int index = find_key(pName); 981 | return (index < 0) ? NULL : &get_object()[index].get_value(); 982 | } 983 | 984 | inline int find_key(const char *pName) const 985 | { 986 | if (!is_object()) 987 | { 988 | PJSON_ASSERT(0); 989 | return -1; 990 | } 991 | 992 | const uint n = get_array().size(); 993 | const key_value_vec_t &obj = get_object(); 994 | 995 | for (uint i = 0; i < n; i++) 996 | if (strcmp(pName, obj[i].get_key().get_ptr("")) == 0) 997 | return i; 998 | 999 | return -1; 1000 | } 1001 | 1002 | inline bool has_key(const char *pName) const 1003 | { 1004 | return find_key(pName) >= 0; 1005 | } 1006 | 1007 | inline bool as_bool(const char *pName, bool def = false) const 1008 | { 1009 | int index = find_key(pName); 1010 | return (index < 0) ? def : get_object()[index].get_value().as_bool(def); 1011 | } 1012 | 1013 | inline int as_int32(const char *pName, int32 def = 0) const 1014 | { 1015 | int index = find_key(pName); 1016 | return (index < 0) ? def : get_object()[index].get_value().as_int32(def); 1017 | } 1018 | 1019 | inline int64 as_int64(const char *pName, int64 def = 0) const 1020 | { 1021 | int index = find_key(pName); 1022 | return (index < 0) ? def : get_object()[index].get_value().as_int64(def); 1023 | } 1024 | 1025 | inline float as_float(const char *pName, float def = 0.0f) const 1026 | { 1027 | int index = find_key(pName); 1028 | return (index < 0) ? def : get_object()[index].get_value().as_float(def); 1029 | } 1030 | 1031 | inline double as_double(const char *pName, double def = 0.0f) const 1032 | { 1033 | int index = find_key(pName); 1034 | return (index < 0) ? def : get_object()[index].get_value().as_double(def); 1035 | } 1036 | 1037 | inline const char* as_string_ptr(const char *pName, const char *pDef = "") const 1038 | { 1039 | int index = find_key(pName); 1040 | if (index < 0) 1041 | return pDef; 1042 | const char *p = get_object()[index].get_value().as_string_ptr(); 1043 | return p ? p : pDef; 1044 | } 1045 | 1046 | inline value_variant& get_value_at_index(uint index) { PJSON_ASSERT(is_object_or_array()); return is_object() ? get_object()[index].get_value() : get_array()[index]; } 1047 | inline const value_variant& get_value_at_index(uint index) const { PJSON_ASSERT(is_object_or_array()); return is_object() ? get_object()[index].get_value() : get_array()[index]; } 1048 | 1049 | inline value_variant& operator[](uint index) { PJSON_ASSERT(is_object_or_array()); return is_object() ? get_object()[index].get_value() : get_array()[index]; } 1050 | inline const value_variant& operator[](uint index) const { PJSON_ASSERT(is_object_or_array()); return is_object() ? get_object()[index].get_value() : get_array()[index]; } 1051 | 1052 | inline json_value_type_t get_value_type_at_index(uint index) const { return get_value_at_index(index).get_type(); } 1053 | 1054 | inline bool is_child_at_index(uint index) const { return get_value_type_at_index(index) >= cJSONValueTypeArray; } 1055 | 1056 | inline bool has_children() const 1057 | { 1058 | if (is_object()) 1059 | { 1060 | const key_value_vec_t& obj = get_object(); 1061 | const uint s = obj.size(); 1062 | for (uint i = 0; i < s; ++i) 1063 | if (obj[i].get_value().is_object_or_array()) 1064 | return true; 1065 | } 1066 | else if (is_array()) 1067 | { 1068 | const value_variant_vec_t& arr = get_array(); 1069 | const uint s = arr.size(); 1070 | for (uint i = 0; i < s; ++i) 1071 | if (arr[i].is_object_or_array()) 1072 | return true; 1073 | } 1074 | return false; 1075 | } 1076 | 1077 | inline void clear_object_or_array() 1078 | { 1079 | PJSON_ASSERT(is_object_or_array()); 1080 | if (is_object()) 1081 | get_object().clear(); 1082 | else if (is_array()) 1083 | get_array().clear(); 1084 | } 1085 | 1086 | inline void resize(uint n, pool_allocator& alloc) 1087 | { 1088 | PJSON_ASSERT(is_object_or_array()); 1089 | if (is_object()) 1090 | get_object().resize(n, alloc); 1091 | else if (is_array()) 1092 | get_array().resize(n, alloc); 1093 | } 1094 | 1095 | inline void set_key_name_at_index(uint index, const char *pKey, uint key_len, pool_allocator& alloc) 1096 | { 1097 | PJSON_ASSERT(is_object()); 1098 | string_vec_t& str = get_object()[index].get_key(); 1099 | str.assign(pKey, key_len + 1, alloc); 1100 | } 1101 | 1102 | inline void set_key_name_at_index(uint index, const char *pKey, pool_allocator& alloc) 1103 | { 1104 | set_key_name_at_index(index, pKey, static_cast(strlen(pKey)) + 1, alloc); 1105 | } 1106 | 1107 | inline value_variant& add_key_value(const char* pKey, uint key_len, const value_variant& val, pool_allocator& alloc) 1108 | { 1109 | PJSON_ASSERT(is_object()); 1110 | key_value_vec_t& obj = get_object(); 1111 | key_value_t* pKey_value = obj.enlarge_no_construct(1, alloc); 1112 | pKey_value->get_key().construct(pKey, key_len + 1, alloc); 1113 | pKey_value->get_value().construct(val, alloc); 1114 | return *this; 1115 | } 1116 | 1117 | inline value_variant& add_key_value(const char* pKey, const value_variant& val, pool_allocator& alloc) 1118 | { 1119 | return add_key_value(pKey, static_cast(strlen(pKey)), val, alloc); 1120 | } 1121 | 1122 | inline value_variant& add_value(const value_variant& val, pool_allocator& alloc) 1123 | { 1124 | PJSON_ASSERT(is_array()); 1125 | get_array().enlarge_no_construct(1, alloc)->construct(val, alloc); 1126 | return *this; 1127 | } 1128 | 1129 | bool serialize(char* pBuf, size_t buf_size, size_t* pSize = NULL, bool formatted = true, bool null_terminate = true) const 1130 | { 1131 | serialize_helper helper(pBuf, buf_size); 1132 | serialize_internal(helper, formatted, null_terminate, 0); 1133 | if (pSize) 1134 | *pSize = helper.size(); 1135 | return (helper.size() < buf_size); 1136 | } 1137 | 1138 | bool serialize(char_vec_t& buf, bool formatted = true, bool null_terminate = true) const 1139 | { 1140 | serialize_helper helper(buf); 1141 | serialize_internal(helper, formatted, null_terminate, 0); 1142 | return true; 1143 | } 1144 | 1145 | protected: 1146 | // Manual constructor 1147 | inline void construct(json_value_type_t type) 1148 | { 1149 | m_type = type; 1150 | memset(&m_data, 0, sizeof(m_data)); 1151 | } 1152 | 1153 | // Assumes variant has NOT been constructed yet. 1154 | inline void construct(const value_variant& other, pool_allocator& alloc) 1155 | { 1156 | m_type = other.m_type; 1157 | m_data.m_nVal = other.m_data.m_nVal; 1158 | if (m_type >= cJSONValueTypeString) 1159 | { 1160 | if (m_type == cJSONValueTypeObject) 1161 | get_object().construct(other.get_object(), alloc); 1162 | else if (m_type == cJSONValueTypeArray) 1163 | get_array().construct(other.get_array(), alloc); 1164 | else 1165 | get_string().construct(other.get_string(), alloc); 1166 | } 1167 | } 1168 | 1169 | inline bool convert_to_bool(bool& val, bool def) const 1170 | { 1171 | switch (m_type) 1172 | { 1173 | case cJSONValueTypeBool: 1174 | case cJSONValueTypeInt: 1175 | { 1176 | val = (m_data.m_nVal != 0); 1177 | return true; 1178 | } 1179 | case cJSONValueTypeDouble: 1180 | { 1181 | val = (m_data.m_flVal != 0); 1182 | return true; 1183 | } 1184 | case cJSONValueTypeString: 1185 | { 1186 | if (!pjson_stricmp(get_string_ptr(), "false")) 1187 | { 1188 | val = false; 1189 | return true; 1190 | } 1191 | else if (!pjson_stricmp(get_string_ptr(), "true")) 1192 | { 1193 | val = true; 1194 | return true; 1195 | } 1196 | val = (atof(get_string_ptr()) != 0.0f); 1197 | return true; 1198 | } 1199 | default: 1200 | break; 1201 | } 1202 | val = def; 1203 | return false; 1204 | } 1205 | 1206 | inline bool convert_to_int32(int32& val, int32 def) const 1207 | { 1208 | val = def; 1209 | int64 val64; 1210 | if (!convert_to_int64(val64, def)) 1211 | return false; 1212 | if ((val64 < std::numeric_limits::min()) || (val64 > std::numeric_limits::max())) 1213 | return false; 1214 | val = static_cast(val64); 1215 | return true; 1216 | } 1217 | 1218 | inline bool convert_to_int64(int64& val, int64 def) const 1219 | { 1220 | switch (m_type) 1221 | { 1222 | case cJSONValueTypeBool: 1223 | case cJSONValueTypeInt: 1224 | { 1225 | val = m_data.m_nVal; 1226 | return true; 1227 | } 1228 | case cJSONValueTypeDouble: 1229 | { 1230 | val = static_cast(m_data.m_flVal); 1231 | return true; 1232 | } 1233 | case cJSONValueTypeString: 1234 | { 1235 | if (!pjson_stricmp(get_string_ptr(), "false")) 1236 | { 1237 | val = 0; 1238 | return true; 1239 | } 1240 | else if (!pjson_stricmp(get_string_ptr(), "true")) 1241 | { 1242 | val = 1; 1243 | return true; 1244 | } 1245 | double flVal = floor(atof(get_string_ptr())); 1246 | if ((flVal >= std::numeric_limits::min()) && (flVal <= std::numeric_limits::max())) 1247 | { 1248 | val = static_cast(flVal); 1249 | return true; 1250 | } 1251 | break; 1252 | } 1253 | default: 1254 | break; 1255 | } 1256 | val = def; 1257 | return false; 1258 | } 1259 | 1260 | inline bool convert_to_float(float& val, float def) const 1261 | { 1262 | switch (m_type) 1263 | { 1264 | case cJSONValueTypeBool: 1265 | case cJSONValueTypeInt: 1266 | { 1267 | val = static_cast(m_data.m_nVal); 1268 | return true; 1269 | } 1270 | case cJSONValueTypeDouble: 1271 | { 1272 | val = static_cast(m_data.m_flVal); 1273 | return true; 1274 | } 1275 | case cJSONValueTypeString: 1276 | { 1277 | if (!pjson_stricmp(get_string_ptr(), "false")) 1278 | { 1279 | val = 0; 1280 | return true; 1281 | } 1282 | else if (!pjson_stricmp(get_string_ptr(), "true")) 1283 | { 1284 | val = 1; 1285 | return true; 1286 | } 1287 | val = static_cast(atof(get_string_ptr())); 1288 | return true; 1289 | } 1290 | default: 1291 | break; 1292 | } 1293 | val = def; 1294 | return false; 1295 | } 1296 | 1297 | inline bool convert_to_double(double& val, double def) const 1298 | { 1299 | switch (m_type) 1300 | { 1301 | case cJSONValueTypeBool: 1302 | case cJSONValueTypeInt: 1303 | { 1304 | val = static_cast(m_data.m_nVal); 1305 | return true; 1306 | } 1307 | case cJSONValueTypeDouble: 1308 | { 1309 | val = m_data.m_flVal; 1310 | return true; 1311 | } 1312 | case cJSONValueTypeString: 1313 | { 1314 | if (!pjson_stricmp(get_string_ptr(), "false")) 1315 | { 1316 | val = 0; 1317 | return true; 1318 | } 1319 | else if (!pjson_stricmp(get_string_ptr(), "true")) 1320 | { 1321 | val = 1; 1322 | return true; 1323 | } 1324 | val = atof(get_string_ptr()); 1325 | return true; 1326 | } 1327 | default: 1328 | break; 1329 | } 1330 | val = def; 1331 | return false; 1332 | } 1333 | 1334 | inline bool convert_to_string(char* pBuf, size_t buf_size) const 1335 | { 1336 | switch (m_type) 1337 | { 1338 | case cJSONValueTypeNull: 1339 | { 1340 | pBuf[0] = 'n', pBuf[1] = 'u', pBuf[2] = 'l', pBuf[3] = 'l', pBuf[4] = '\0'; 1341 | return true; 1342 | } 1343 | case cJSONValueTypeBool: 1344 | { 1345 | if (m_data.m_nVal) 1346 | pBuf[0] = 't', pBuf[1] = 'r', pBuf[2] = 'u', pBuf[3] = 'e', pBuf[4] = '\0'; 1347 | else 1348 | pBuf[0] = 'f', pBuf[1] = 'a', pBuf[2] = 'l', pBuf[3] = 's', pBuf[4] = 'e', pBuf[5] = '\0'; 1349 | return true; 1350 | } 1351 | case cJSONValueTypeInt: 1352 | { 1353 | char* pDst = pBuf; 1354 | int64 n = m_data.m_nVal; 1355 | 1356 | uint64 s = static_cast(n >> 63); 1357 | *pDst = '-'; 1358 | pDst -= s; 1359 | n = (n ^ s) - s; 1360 | 1361 | char* pLeft = pDst; 1362 | 1363 | do 1364 | { 1365 | *pDst++ = '0' + (n % 10); 1366 | n /= 10; 1367 | } while (n); 1368 | 1369 | *pDst = '\0'; 1370 | 1371 | do 1372 | { 1373 | char c = *--pDst; 1374 | *pDst = *pLeft; 1375 | *pLeft++ = c; 1376 | } while (pDst > pLeft); 1377 | return true; 1378 | } 1379 | case cJSONValueTypeDouble: 1380 | { 1381 | #ifdef _MSC_VER 1382 | return 0 == _gcvt_s(pBuf, buf_size, m_data.m_flVal, 15); 1383 | #else // TODO 1384 | abort(); 1385 | #endif 1386 | } 1387 | default: 1388 | break; 1389 | } 1390 | return false; 1391 | } 1392 | 1393 | inline bool convert_to_string(string_t& val, const char* pDef) const 1394 | { 1395 | char buf[64]; 1396 | if (m_type == cJSONValueTypeString) 1397 | val = get_string_ptr(); 1398 | else 1399 | { 1400 | if (!convert_to_string(buf, sizeof(buf))) 1401 | { 1402 | val.assign(pDef); 1403 | return false; 1404 | } 1405 | val.assign(buf); 1406 | } 1407 | return true; 1408 | } 1409 | 1410 | inline uint8 get_end_char() const { return (m_type == cJSONValueTypeArray) ? ']' : '}'; } 1411 | 1412 | template 1413 | void serialize_node(serializer& out, bool formatted, uint cur_indent) const 1414 | { 1415 | char buf[64]; 1416 | const uint size = get_array().size(); 1417 | 1418 | if (!size) 1419 | { 1420 | static const char* g_empty_object_strs[4] = { "[]", "[ ]", "{}", "{ }" }; 1421 | out.puts(g_empty_object_strs[is_object() * 2 + formatted], 2 + formatted); 1422 | return; 1423 | } 1424 | 1425 | if (formatted && is_array() && !has_children()) 1426 | { 1427 | size_t start_of_line_ofs = out.size(); 1428 | out.puts("[ ", 2); 1429 | 1430 | const uint cMaxLineLen = 100; 1431 | 1432 | for (uint i = 0; i < size; i++) 1433 | { 1434 | const value_variant& child_val = get_value_at_index(i); 1435 | if (child_val.is_string()) 1436 | out.print_escaped(child_val.get_string()); 1437 | else 1438 | { 1439 | child_val.convert_to_string(buf, sizeof(buf)); 1440 | out.puts(buf); 1441 | } 1442 | 1443 | if (i != size - 1) 1444 | out.puts(", ", 2); 1445 | 1446 | if (((out.size() - start_of_line_ofs) > cMaxLineLen) && (i != size - 1)) 1447 | { 1448 | out.print_char('\n'); 1449 | out.print_tabs(cur_indent + 1); 1450 | 1451 | start_of_line_ofs = out.size(); 1452 | } 1453 | } 1454 | 1455 | out.puts(" ]", 2); 1456 | return; 1457 | } 1458 | 1459 | out.print_char(is_object() ? '{' : '['); 1460 | if (formatted) 1461 | out.print_char('\n'); 1462 | 1463 | cur_indent++; 1464 | 1465 | for (uint i = 0; i < size; i++) 1466 | { 1467 | const value_variant& child_val = get_value_at_index(i); 1468 | 1469 | if (formatted) 1470 | out.print_tabs(cur_indent); 1471 | 1472 | if (is_object()) 1473 | { 1474 | out.print_escaped(get_object()[i].get_key()); 1475 | if (formatted) 1476 | out.puts(" : ", 3); 1477 | else 1478 | out.print_char(':'); 1479 | } 1480 | 1481 | json_value_type_t val_type = child_val.get_type(); 1482 | if (val_type >= cJSONValueTypeArray) 1483 | child_val.serialize_node(out, formatted, cur_indent); 1484 | else if (val_type == cJSONValueTypeString) 1485 | out.print_escaped(child_val.get_string()); 1486 | else 1487 | { 1488 | child_val.convert_to_string(buf, sizeof(buf)); 1489 | out.puts(buf); 1490 | } 1491 | 1492 | if (i != size - 1) 1493 | out.print_char(','); 1494 | if (formatted) 1495 | out.print_char('\n'); 1496 | } 1497 | 1498 | cur_indent--; 1499 | 1500 | if (formatted) 1501 | out.print_tabs(cur_indent); 1502 | out.print_char(is_object() ? '}' : ']'); 1503 | } 1504 | 1505 | template 1506 | void serialize_internal(serializer& out, bool formatted, bool null_terminate, uint cur_indent) const 1507 | { 1508 | if (formatted) 1509 | out.print_tabs(cur_indent); 1510 | if (is_object_or_array()) 1511 | { 1512 | serialize_node(out, formatted, cur_indent); 1513 | if (formatted) 1514 | out.print_char('\n'); 1515 | } 1516 | else 1517 | { 1518 | if (is_string()) 1519 | out.print_escaped(get_string()); 1520 | else 1521 | { 1522 | string_t str; 1523 | if (get_string_value(str)) 1524 | out.puts(str.c_str(), str.length()); 1525 | } 1526 | } 1527 | if (null_terminate) 1528 | out.print_char('\0'); 1529 | } 1530 | 1531 | template value_variant(T*); 1532 | template value_variant(const T*); 1533 | template value_variant& operator= (T*); 1534 | template value_variant& operator= (const T*); 1535 | }; 1536 | #pragma pack(pop) 1537 | 1538 | inline key_value_t::key_value_t(const key_value_t& other, pool_allocator& alloc) : 1539 | m_key(other.get_key(), alloc) 1540 | { 1541 | get_value().construct(other.get_value(), alloc); 1542 | } 1543 | 1544 | inline void key_value_t::assign(const key_value_t& src, pool_allocator& alloc) 1545 | { 1546 | get_key().assign(src.get_key(), alloc); 1547 | get_value().assign(src.get_value(), alloc); 1548 | } 1549 | 1550 | // ---- class error_info 1551 | 1552 | class error_info 1553 | { 1554 | public: 1555 | inline error_info() : m_ofs(0), m_pError_message(NULL) { } 1556 | inline void set(size_t ofs, const char* pMsg) { m_ofs = ofs; m_pError_message = pMsg; } 1557 | 1558 | size_t m_ofs; 1559 | const char* m_pError_message; 1560 | }; 1561 | 1562 | // ---- class growable_stack 1563 | 1564 | class growable_stack 1565 | { 1566 | public: 1567 | inline growable_stack(uint initial_size) : 1568 | m_pBuf(NULL), 1569 | m_size(initial_size), 1570 | m_ofs(0) 1571 | { 1572 | if (initial_size) 1573 | m_pBuf = static_cast(pjson_malloc(initial_size)); 1574 | } 1575 | 1576 | inline ~growable_stack() 1577 | { 1578 | pjson_free(m_pBuf); 1579 | } 1580 | 1581 | inline void clear() 1582 | { 1583 | pjson_free(m_pBuf); 1584 | m_pBuf = NULL; 1585 | m_size = 0; 1586 | m_ofs = 0; 1587 | } 1588 | 1589 | inline uint8* get_top_ptr() { return reinterpret_cast(m_pBuf) + m_ofs; } 1590 | 1591 | template inline T* get_top_obj() { return reinterpret_cast(m_pBuf + m_ofs - sizeof(T)); } 1592 | 1593 | inline void reset() { m_ofs = 0; } 1594 | 1595 | inline size_t get_ofs() { return m_ofs; } 1596 | 1597 | template 1598 | PJSON_FORCEINLINE T* push(uint num) 1599 | { 1600 | const size_t bytes_needed = sizeof(T) * num; 1601 | T* pResult = reinterpret_cast(m_pBuf + m_ofs); 1602 | m_ofs += bytes_needed; 1603 | 1604 | if (m_ofs > m_size) 1605 | { 1606 | m_ofs -= bytes_needed; 1607 | 1608 | m_size = PJSON_MAX(1, m_size * 2); 1609 | while(m_size <= (m_ofs + bytes_needed)) 1610 | m_size *= 2; 1611 | m_pBuf = static_cast(pjson_realloc(m_pBuf, m_size)); 1612 | 1613 | pResult = reinterpret_cast(m_pBuf + m_ofs); 1614 | m_ofs += bytes_needed; 1615 | } 1616 | 1617 | PJSON_ASSERT(m_ofs <= m_size); 1618 | return pResult; 1619 | } 1620 | 1621 | template 1622 | inline T* pop(uint num) 1623 | { 1624 | size_t bytes_needed = sizeof(T) * num; 1625 | PJSON_ASSERT(bytes_needed <= m_ofs); 1626 | m_ofs -= bytes_needed; 1627 | return reinterpret_cast(m_pBuf + m_ofs); 1628 | } 1629 | 1630 | private: 1631 | uint8* m_pBuf; 1632 | size_t m_size; 1633 | size_t m_ofs; 1634 | }; 1635 | 1636 | // ---- class document 1637 | 1638 | class document : public value_variant 1639 | { 1640 | document(const document&); 1641 | document& operator= (const document&); 1642 | 1643 | public: 1644 | inline document(uint initial_pool_size = 0, uint min_pool_chunk_size = PJSON_DEFAULT_MIN_CHUNK_SIZE, uint initial_stack_size = 0) : 1645 | m_allocator(initial_pool_size, min_pool_chunk_size), 1646 | m_initial_stack_size(initial_stack_size), 1647 | m_stack(initial_stack_size) 1648 | { 1649 | #if PJSON_PARSE_STATS 1650 | m_parse_stats.clear(); 1651 | #endif 1652 | } 1653 | 1654 | inline ~document() 1655 | { 1656 | } 1657 | 1658 | const pool_allocator& get_allocator() const { return m_allocator; } 1659 | pool_allocator& get_allocator() { return m_allocator; } 1660 | 1661 | void clear() 1662 | { 1663 | set_to_null(); 1664 | m_allocator.clear(); 1665 | m_stack.clear(); 1666 | #if PJSON_PARSE_STATS 1667 | m_parse_stats.clear(); 1668 | #endif 1669 | } 1670 | 1671 | bool deserialize_in_place(char* pStr) 1672 | { 1673 | return deserialize_start((uint8*)pStr); 1674 | } 1675 | 1676 | #if PJSON_PARSE_STATS 1677 | struct parse_stats_t 1678 | { 1679 | size_t m_num_string, m_num_string_chars; 1680 | size_t m_num_numeric, m_num_numeric_chars; 1681 | size_t m_num_whitespace_blocks, m_num_whitespace_chars; 1682 | size_t m_num_control; 1683 | size_t m_num_comment, m_num_comment_chars; 1684 | size_t m_num_value_pop, m_value_pop_bytes; 1685 | size_t m_num_bool_chars; 1686 | size_t m_num_escape_breaks; 1687 | size_t m_num_unicode_escapes; 1688 | 1689 | void clear() { memset(this, 0, sizeof(*this)); } 1690 | }; 1691 | 1692 | const parse_stats_t& get_parse_stats() const { return m_parse_stats; } 1693 | parse_stats_t& get_parse_stats() { return m_parse_stats; } 1694 | #endif 1695 | 1696 | private: 1697 | pool_allocator m_allocator; 1698 | uint m_initial_stack_size; 1699 | growable_stack m_stack; 1700 | error_info m_error_info; 1701 | const uint8* m_pStart; 1702 | const uint8* m_pStr; 1703 | 1704 | inline bool set_error(const uint8* pStr, const char* pMsg) 1705 | { 1706 | m_pStr = pStr; 1707 | m_error_info.set(m_pStr - m_pStart, pMsg); 1708 | return false; 1709 | } 1710 | 1711 | #if PJSON_PARSE_STATS 1712 | parse_stats_t m_parse_stats; 1713 | #endif 1714 | 1715 | #if PJSON_PARSE_STATS 1716 | #define PJSON_INCREMENT_STAT(x) do { ++m_parse_stats.x; } while(0) 1717 | #define PJSON_UPDATE_STAT(x, n) do { m_parse_stats.x += n; } while(0) 1718 | #else 1719 | #define PJSON_INCREMENT_STAT(x) do { } while(0) 1720 | #define PJSON_UPDATE_STAT(x, n) do { } while(0) 1721 | #endif 1722 | 1723 | #define PJSON_SKIP_WHITESPACE \ 1724 | while (globals::s_parse_flags[*pStr] & 4) \ 1725 | { \ 1726 | PJSON_INCREMENT_STAT(m_num_whitespace_blocks); \ 1727 | do { \ 1728 | if (!(globals::s_parse_flags[pStr[1]] & 4)) { ++pStr; PJSON_INCREMENT_STAT(m_num_whitespace_chars); break; } \ 1729 | if (!(globals::s_parse_flags[pStr[2]] & 4)) { pStr += 2; PJSON_UPDATE_STAT(m_num_whitespace_chars, 2); break; } \ 1730 | if (!(globals::s_parse_flags[pStr[3]] & 4)) { pStr += 3; PJSON_UPDATE_STAT(m_num_whitespace_chars, 3); break; } \ 1731 | pStr += 4; PJSON_UPDATE_STAT(m_num_whitespace_chars, 4); \ 1732 | } while (globals::s_parse_flags[*pStr] & 4); \ 1733 | if ((*pStr != '/') || (pStr[1] != '/')) break; \ 1734 | pStr += 2; PJSON_INCREMENT_STAT(m_num_comment); PJSON_UPDATE_STAT(m_num_comment_chars, 2); \ 1735 | while ((*pStr) && (*pStr != '\n') && (*pStr != '\r')) { PJSON_INCREMENT_STAT(m_num_comment_chars); ++pStr; } \ 1736 | } 1737 | 1738 | inline const uint8* skip_whitespace(const uint8* p) 1739 | { 1740 | uint8 c; 1741 | while ((c = *p) != '\0') 1742 | { 1743 | if ((c == ' ') || (c == '\t')) 1744 | { 1745 | do 1746 | { 1747 | PJSON_INCREMENT_STAT(m_num_whitespace_chars); 1748 | } while (*++p == c); 1749 | continue; 1750 | } 1751 | 1752 | if ((c == '/') && (p[1] == '/')) 1753 | { 1754 | p += 2; PJSON_INCREMENT_STAT(m_num_comment); PJSON_UPDATE_STAT(m_num_comment_chars, 2); 1755 | while ((*p) && (*p != '\n') && (*p != '\r')) 1756 | { 1757 | PJSON_UPDATE_STAT(m_num_comment_chars, 2); 1758 | ++p; 1759 | } 1760 | continue; 1761 | } 1762 | else if (c > ' ') 1763 | break; 1764 | 1765 | PJSON_INCREMENT_STAT(m_num_whitespace_chars); 1766 | ++p; 1767 | } 1768 | return p; 1769 | } 1770 | 1771 | bool deserialize_internal() 1772 | { 1773 | static const uint8 g_utf8_first_byte[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; 1774 | 1775 | m_stack.reset(); 1776 | memcpy(m_stack.push(1), static_cast(this), sizeof(value_variant)); 1777 | 1778 | const uint8* pStr = ++m_pStr; 1779 | PJSON_UPDATE_STAT(m_num_control, 1); 1780 | 1781 | bool cur_is_object = is_object(); 1782 | uint8 cur_end_char = get_end_char(); 1783 | uint cur_num_elements = 0; 1784 | 1785 | for ( ; ; ) 1786 | { 1787 | PJSON_SKIP_WHITESPACE; 1788 | 1789 | uint8 c = *pStr; 1790 | 1791 | if (c == ',') 1792 | { 1793 | if (!cur_num_elements) 1794 | return set_error(pStr, "Unexpected comma"); 1795 | 1796 | ++pStr; 1797 | PJSON_UPDATE_STAT(m_num_control, 1); 1798 | PJSON_SKIP_WHITESPACE; 1799 | c = *pStr; 1800 | } 1801 | else if ((cur_num_elements) && (c != cur_end_char)) 1802 | return set_error(pStr, "Expected comma or object/array end character"); 1803 | 1804 | while (c == cur_end_char) 1805 | { 1806 | PJSON_UPDATE_STAT(m_num_control, 1); 1807 | ++pStr; 1808 | 1809 | for ( ; ; ) 1810 | { 1811 | uint n = cur_num_elements, num_bytes = cur_num_elements * (cur_is_object ? sizeof(key_value_t) : sizeof(value_variant)); 1812 | void* pSrc = m_stack.pop(num_bytes); 1813 | PJSON_INCREMENT_STAT(m_num_value_pop); 1814 | PJSON_UPDATE_STAT(m_value_pop_bytes, num_bytes); 1815 | 1816 | // The top of the stack (after popping the current array/object) could contain either a value_variant (if cur_is_object is set), 1817 | // or a key_value_t, which ends in a value_variant. So all we need to do is look at the very end, which always has a value_variant. 1818 | value_variant* pCur_variant = m_stack.get_top_obj(); 1819 | 1820 | value_variant_vec_t& arr = pCur_variant->get_array(); 1821 | cur_is_object = (arr.m_p != NULL); 1822 | cur_end_char = cur_is_object ? '}' : ']'; 1823 | cur_num_elements = arr.m_size; 1824 | 1825 | arr.m_size = n; 1826 | arr.m_p = NULL; 1827 | if (num_bytes) 1828 | memcpy(arr.m_p = static_cast(m_allocator.Alloc(num_bytes)), pSrc, num_bytes); 1829 | 1830 | if (m_stack.get_ofs() <= sizeof(value_variant)) 1831 | { 1832 | PJSON_ASSERT(m_stack.get_ofs() == sizeof(value_variant)); 1833 | memcpy(static_cast(this), m_stack.pop(1), sizeof(value_variant)); 1834 | m_pStr = pStr; 1835 | return true; 1836 | } 1837 | 1838 | PJSON_SKIP_WHITESPACE; 1839 | 1840 | if (*pStr == ',') 1841 | { 1842 | PJSON_UPDATE_STAT(m_num_control, 1); 1843 | ++pStr; 1844 | 1845 | PJSON_SKIP_WHITESPACE; 1846 | 1847 | c = *pStr; 1848 | break; 1849 | } 1850 | 1851 | if (*pStr++ != cur_end_char) 1852 | return set_error(pStr, "Unexpected character within object or array"); 1853 | PJSON_UPDATE_STAT(m_num_control, 1); 1854 | } 1855 | } 1856 | 1857 | ++cur_num_elements; 1858 | 1859 | value_variant* pChild_variant; 1860 | 1861 | if (!cur_is_object) 1862 | pChild_variant = m_stack.push(1); 1863 | else 1864 | { 1865 | if (c != '\"') 1866 | return set_error(pStr, "Expected quoted key string"); 1867 | 1868 | ++pStr; PJSON_INCREMENT_STAT(m_num_string); PJSON_INCREMENT_STAT(m_num_string_chars); 1869 | 1870 | uint8* pBuf = (uint8*)pStr; 1871 | 1872 | c = *pStr++; PJSON_INCREMENT_STAT(m_num_string_chars); 1873 | if (!(globals::s_parse_flags[c] & 1)) 1874 | { 1875 | do 1876 | { 1877 | c = pStr[0]; if (globals::s_parse_flags[c] & 1) { ++pStr; PJSON_UPDATE_STAT(m_num_string_chars, 1); break; } 1878 | c = pStr[1]; if (globals::s_parse_flags[c] & 1) { pStr += 2; PJSON_UPDATE_STAT(m_num_string_chars, 2); break; } 1879 | c = pStr[2]; if (globals::s_parse_flags[c] & 1) { pStr += 3; PJSON_UPDATE_STAT(m_num_string_chars, 3); break; } 1880 | c = pStr[3]; 1881 | pStr += 4; PJSON_UPDATE_STAT(m_num_string_chars, 4); 1882 | } while (!(globals::s_parse_flags[c] & 1)); 1883 | } 1884 | 1885 | uint8* pDst = (uint8*)pStr - 1; 1886 | 1887 | if (c != '\"') PJSON_INCREMENT_STAT(m_num_escape_breaks); 1888 | 1889 | while (c != '\"') 1890 | { 1891 | if (globals::s_parse_flags[c] & 2) 1892 | return set_error(pStr, "Missing end quote"); 1893 | 1894 | c = *pStr++; PJSON_INCREMENT_STAT(m_num_string_chars); 1895 | if (c == 'u') 1896 | { 1897 | PJSON_INCREMENT_STAT(m_num_unicode_escapes); 1898 | uint u = 0; 1899 | for (uint i = 0; i < 4; i++) 1900 | { 1901 | u <<= 4; 1902 | int cc = *pStr++; PJSON_INCREMENT_STAT(m_num_string_chars); 1903 | if ((cc >= 'A') && (cc <= 'F')) 1904 | u += cc - 'A' + 10; 1905 | else if ((cc >= 'a') && (cc <= 'f')) 1906 | u += cc - 'a' + 10; 1907 | else if ((cc >= '0') && (cc <= '9')) 1908 | u += cc - '0'; 1909 | else 1910 | return set_error(pStr, "Invalid Unicode escape"); 1911 | } 1912 | 1913 | uint len = 3; if ((u) && (u < 0x80)) len = 1; else if (u < 0x800) len = 2; 1914 | 1915 | pDst += len; 1916 | uint8* q = pDst; 1917 | switch (len) 1918 | { 1919 | case 3: *--q = static_cast((u | 0x80) & 0xBF); u >>= 6; // falls through 1920 | case 2: *--q = static_cast((u | 0x80) & 0xBF); u >>= 6; // falls through 1921 | case 1: *--q = static_cast(u | g_utf8_first_byte[len]); 1922 | } 1923 | } 1924 | else 1925 | { 1926 | switch (c) 1927 | { 1928 | case 'b': c = '\b'; break; 1929 | case 'f': c = '\f'; break; 1930 | case 'n': c = '\n'; break; 1931 | case 'r': c = '\r'; break; 1932 | case 't': c = '\t'; break; 1933 | case '\\': case '\"': case '/': break; 1934 | case '\0': return set_error(pStr, "Incomplete string escape"); 1935 | default: { *pDst++ = '\\'; break; } // unrecognized escape, so forcefully escape the backslash (not standard) 1936 | } 1937 | 1938 | *pDst++ = c; 1939 | } 1940 | 1941 | c = *pStr++; PJSON_INCREMENT_STAT(m_num_string_chars); 1942 | while (!(globals::s_parse_flags[c] & 1)) 1943 | { 1944 | pDst[0] = c; 1945 | c = pStr[0]; if (globals::s_parse_flags[c] & 1) { ++pDst; ++pStr; PJSON_INCREMENT_STAT(m_num_string_chars); break; } 1946 | pDst[1] = c; 1947 | c = pStr[1]; if (globals::s_parse_flags[c] & 1) { pDst += 2; pStr += 2; PJSON_UPDATE_STAT(m_num_string_chars, 2); break; } 1948 | pDst[2] = c; 1949 | c = pStr[2]; if (globals::s_parse_flags[c] & 1) { pDst += 3; pStr += 3; PJSON_UPDATE_STAT(m_num_string_chars, 3); break; } 1950 | pDst[3] = c; 1951 | pDst += 4; 1952 | c = pStr[3]; 1953 | pStr += 4; PJSON_UPDATE_STAT(m_num_string_chars, 4); 1954 | } 1955 | } 1956 | 1957 | *pDst++ = '\0'; 1958 | 1959 | key_value_t* pKey_value = m_stack.push(1); 1960 | pChild_variant = &pKey_value->get_value(); 1961 | pKey_value->get_key().m_p = (char*)pBuf; 1962 | pKey_value->get_key().m_size = static_cast(pDst - pBuf - 1); 1963 | 1964 | PJSON_SKIP_WHITESPACE; 1965 | 1966 | if (*pStr != ':') 1967 | return set_error(pStr, "Missing colon after key"); 1968 | 1969 | ++pStr; PJSON_INCREMENT_STAT(m_num_control); 1970 | PJSON_SKIP_WHITESPACE; 1971 | 1972 | c = *pStr; 1973 | } 1974 | 1975 | switch (c) 1976 | { 1977 | case '{': 1978 | case '[': 1979 | { 1980 | ++pStr; PJSON_INCREMENT_STAT(m_num_control); 1981 | 1982 | pChild_variant->m_type = (c == '{') ? cJSONValueTypeObject : cJSONValueTypeArray; 1983 | pChild_variant->m_data.m_object.m_size = cur_num_elements; 1984 | pChild_variant->m_data.m_object.m_p = (key_value_t*)cur_is_object; 1985 | 1986 | cur_is_object = (c == '{'); 1987 | cur_num_elements = 0; 1988 | cur_end_char = c + 2; 1989 | break; 1990 | } 1991 | case '\"': 1992 | { 1993 | ++pStr; PJSON_INCREMENT_STAT(m_num_string); PJSON_INCREMENT_STAT(m_num_string_chars); 1994 | 1995 | uint8* pBuf = (uint8*)pStr; 1996 | 1997 | c = *pStr++; PJSON_INCREMENT_STAT(m_num_string_chars); 1998 | if (!(globals::s_parse_flags[c] & 1)) 1999 | { 2000 | do 2001 | { 2002 | c = pStr[0]; if (globals::s_parse_flags[c] & 1) { ++pStr; PJSON_INCREMENT_STAT(m_num_string_chars); break; } 2003 | c = pStr[1]; if (globals::s_parse_flags[c] & 1) { pStr += 2; PJSON_UPDATE_STAT(m_num_string_chars, 2); break; } 2004 | c = pStr[2]; if (globals::s_parse_flags[c] & 1) { pStr += 3; PJSON_UPDATE_STAT(m_num_string_chars, 3); break; } 2005 | c = pStr[3]; 2006 | pStr += 4; PJSON_UPDATE_STAT(m_num_string_chars, 4); 2007 | } while (!(globals::s_parse_flags[c] & 1)); 2008 | } 2009 | 2010 | uint8* pDst = (uint8*)pStr - 1; 2011 | 2012 | if (c != '\"') PJSON_INCREMENT_STAT(m_num_escape_breaks); 2013 | 2014 | while (c != '\"') 2015 | { 2016 | if (globals::s_parse_flags[c] & 2) 2017 | return set_error(pStr, "Missing end quote"); 2018 | 2019 | c = *pStr++; PJSON_INCREMENT_STAT(m_num_string_chars); 2020 | if (c == 'u') 2021 | { 2022 | PJSON_INCREMENT_STAT(m_num_unicode_escapes); 2023 | uint u = 0; 2024 | for (uint i = 0; i < 4; i++) 2025 | { 2026 | u <<= 4; 2027 | int cc = *pStr++; PJSON_INCREMENT_STAT(m_num_string_chars); 2028 | if ((cc >= 'A') && (cc <= 'F')) 2029 | u += cc - 'A' + 10; 2030 | else if ((cc >= 'a') && (cc <= 'f')) 2031 | u += cc - 'a' + 10; 2032 | else if ((cc >= '0') && (cc <= '9')) 2033 | u += cc - '0'; 2034 | else 2035 | return set_error(pStr, "Invalid Unicode escape"); 2036 | } 2037 | 2038 | uint len = 3; if ((u) && (u < 0x80)) len = 1; else if (u < 0x800) len = 2; 2039 | 2040 | pDst += len; 2041 | uint8* q = pDst; 2042 | switch (len) 2043 | { 2044 | case 3: *--q = static_cast((u | 0x80) & 0xBF); u >>= 6; // falls through 2045 | case 2: *--q = static_cast((u | 0x80) & 0xBF); u >>= 6; // falls through 2046 | case 1: *--q = static_cast(u | g_utf8_first_byte[len]); 2047 | } 2048 | } 2049 | else 2050 | { 2051 | switch (c) 2052 | { 2053 | case 'b': c = '\b'; break; 2054 | case 'f': c = '\f'; break; 2055 | case 'n': c = '\n'; break; 2056 | case 'r': c = '\r'; break; 2057 | case 't': c = '\t'; break; 2058 | case '\\': case '\"': case '/': break; 2059 | case '\0': return set_error(pStr, "Incomplete string escape"); 2060 | default: { *pDst++ = '\\'; break; } // unrecognized escape, so forcefully escape the backslash (not standard) 2061 | } 2062 | 2063 | *pDst++ = c; 2064 | } 2065 | 2066 | c = *pStr++; PJSON_INCREMENT_STAT(m_num_string_chars); 2067 | while (!(globals::s_parse_flags[c] & 1)) 2068 | { 2069 | pDst[0] = c; 2070 | c = pStr[0]; if (globals::s_parse_flags[c] & 1) { ++pDst; ++pStr; PJSON_INCREMENT_STAT(m_num_string_chars); break; } 2071 | pDst[1] = c; 2072 | c = pStr[1]; if (globals::s_parse_flags[c] & 1) { pDst += 2; pStr += 2; PJSON_UPDATE_STAT(m_num_string_chars, 2); break; } 2073 | pDst[2] = c; 2074 | c = pStr[2]; if (globals::s_parse_flags[c] & 1) { pDst += 3; pStr += 3; PJSON_UPDATE_STAT(m_num_string_chars, 3); break; } 2075 | pDst[3] = c; 2076 | pDst += 4; 2077 | c = pStr[3]; 2078 | pStr += 4; PJSON_UPDATE_STAT(m_num_string_chars, 4); 2079 | } 2080 | } 2081 | 2082 | *pDst++ = '\0'; 2083 | 2084 | pChild_variant->m_type = cJSONValueTypeString; 2085 | 2086 | string_vec_t& str = pChild_variant->get_string(); 2087 | str.m_p = (char*)pBuf; 2088 | str.m_size = static_cast(pDst - pBuf - 1); 2089 | 2090 | break; 2091 | } 2092 | case 'n': 2093 | { 2094 | if ((pStr[1] == 'u') && (pStr[2] == 'l') && (pStr[3] == 'l')) 2095 | { 2096 | pStr += 4; PJSON_UPDATE_STAT(m_num_bool_chars, 4); 2097 | pChild_variant->construct(cJSONValueTypeNull); 2098 | } 2099 | else 2100 | return set_error(pStr, "Unrecognized character"); 2101 | break; 2102 | } 2103 | case 't': 2104 | { 2105 | if ((pStr[1] == 'r') && (pStr[2] == 'u') && (pStr[3] == 'e')) 2106 | { 2107 | pStr += 4; PJSON_UPDATE_STAT(m_num_bool_chars, 4); 2108 | pChild_variant->construct(cJSONValueTypeBool); 2109 | pChild_variant->m_data.m_nVal = 1; 2110 | } 2111 | else 2112 | return set_error(pStr, "Unrecognized character"); 2113 | break; 2114 | } 2115 | case 'f': 2116 | { 2117 | if ((pStr[1] == 'a') && (pStr[2] == 'l') && (pStr[3] == 's') && (pStr[4] == 'e')) 2118 | { 2119 | pStr += 5; PJSON_UPDATE_STAT(m_num_bool_chars, 5); 2120 | pChild_variant->construct(cJSONValueTypeBool); 2121 | } 2122 | else 2123 | return set_error(pStr, "Unrecognized character"); 2124 | break; 2125 | } 2126 | case '0': case '1': case '2': case '3': case '4': case '5': 2127 | case '6': case '7': case '8': case '9': case '-': case '.': 2128 | { 2129 | PJSON_INCREMENT_STAT(m_num_numeric); 2130 | if (c == '-') PJSON_INCREMENT_STAT(m_num_numeric_chars); 2131 | 2132 | uint32 n32 = 0; 2133 | int is_neg = (c == '-'); 2134 | c = *(pStr += is_neg); 2135 | 2136 | if (globals::s_parse_flags[c] & 8) 2137 | { 2138 | n32 = c - '0'; c = *++pStr; PJSON_UPDATE_STAT(m_num_numeric_chars, 1); 2139 | if (globals::s_parse_flags[c] & 8) 2140 | { 2141 | n32 = (n32 * 10U) + (c - '0'); c = *++pStr; PJSON_UPDATE_STAT(m_num_numeric_chars, 1); 2142 | if (globals::s_parse_flags[c] & 8) 2143 | { 2144 | n32 = (n32 * 10U) + (c - '0'); c = *++pStr; PJSON_UPDATE_STAT(m_num_numeric_chars, 1); 2145 | if (globals::s_parse_flags[c] & 8) 2146 | { 2147 | n32 = (n32 * 10U) + (c - '0'); c = *++pStr; PJSON_UPDATE_STAT(m_num_numeric_chars, 1); 2148 | if (globals::s_parse_flags[c] & 8) 2149 | { 2150 | n32 = (n32 * 10U) + (c - '0'); c = *++pStr; PJSON_UPDATE_STAT(m_num_numeric_chars, 1); 2151 | if (globals::s_parse_flags[c] & 8) 2152 | { 2153 | n32 = (n32 * 10U) + (c - '0'); c = *++pStr; PJSON_UPDATE_STAT(m_num_numeric_chars, 1); 2154 | if (globals::s_parse_flags[c] & 8) 2155 | { 2156 | n32 = (n32 * 10U) + (c - '0'); c = *++pStr; PJSON_UPDATE_STAT(m_num_numeric_chars, 1); 2157 | if (globals::s_parse_flags[c] & 8) 2158 | { 2159 | n32 = (n32 * 10U) + (c - '0'); c = *++pStr; PJSON_UPDATE_STAT(m_num_numeric_chars, 1); 2160 | } 2161 | } 2162 | } 2163 | } 2164 | } 2165 | } 2166 | } 2167 | } 2168 | 2169 | if (!(globals::s_parse_flags[c] & 0x10)) 2170 | { 2171 | pChild_variant->m_type = cJSONValueTypeInt; 2172 | pChild_variant->m_data.m_nVal = is_neg + (static_cast(n32) ^ (-is_neg)); 2173 | } 2174 | else 2175 | { 2176 | uint64 n64 = n32; 2177 | while (globals::s_parse_flags[c] & 8) 2178 | { 2179 | n64 = n64 * 10U + (c - '0'); PJSON_INCREMENT_STAT(m_num_numeric_chars); c = *++pStr; 2180 | 2181 | if ((!(globals::s_parse_flags[c] & 8)) || (n64 > 0xCCCCCCCCCCCCCCBULL)) 2182 | break; 2183 | 2184 | n64 = n64 * 10U + (c - '0'); PJSON_INCREMENT_STAT(m_num_numeric_chars); c = *++pStr; 2185 | 2186 | if (n64 > 0xCCCCCCCCCCCCCCBULL) 2187 | break; 2188 | } 2189 | 2190 | if (!(globals::s_parse_flags[c] & 0x10)) 2191 | { 2192 | pChild_variant->m_type = cJSONValueTypeInt; 2193 | pChild_variant->m_data.m_nVal = is_neg + (static_cast(n64) ^ (-is_neg)); 2194 | } 2195 | else 2196 | { 2197 | double f = static_cast(n64); 2198 | int scale = 0, escalesign = 1, escale = 0; 2199 | 2200 | while (globals::s_parse_flags[c] & 8) 2201 | { 2202 | f = f * 10.0f + (c - '0'); PJSON_INCREMENT_STAT(m_num_numeric_chars); c = *++pStr; 2203 | if (!(globals::s_parse_flags[c] & 8)) 2204 | break; 2205 | 2206 | f = f * 10.0f + (c - '0'); PJSON_INCREMENT_STAT(m_num_numeric_chars); c = *++pStr; 2207 | } 2208 | 2209 | if (c == '.') 2210 | { 2211 | PJSON_INCREMENT_STAT(m_num_numeric_chars); c = *++pStr; 2212 | while (globals::s_parse_flags[c] & 8) 2213 | { 2214 | scale--; f = f * 10.0f + (c - '0'); PJSON_INCREMENT_STAT(m_num_numeric_chars); c = *++pStr; 2215 | 2216 | if (!(globals::s_parse_flags[c] & 8)) 2217 | break; 2218 | 2219 | scale--; f = f * 10.0f + (c - '0'); PJSON_INCREMENT_STAT(m_num_numeric_chars); c = *++pStr; 2220 | } 2221 | } 2222 | 2223 | if ((c == 'e') || (c == 'E')) 2224 | { 2225 | PJSON_INCREMENT_STAT(m_num_numeric_chars); 2226 | c = *++pStr; 2227 | if (c == '-') 2228 | { 2229 | escalesign = -1; 2230 | PJSON_INCREMENT_STAT(m_num_numeric_chars); c = *++pStr; 2231 | } 2232 | else if (c == '+') 2233 | { 2234 | PJSON_INCREMENT_STAT(m_num_numeric_chars); c = *++pStr; 2235 | } 2236 | while (globals::s_parse_flags[c] & 8) 2237 | { 2238 | if (escale > 0xCCCCCCB) 2239 | return set_error(pStr, "Failed parsing numeric value"); 2240 | escale = escale * 10 + (c - '0'); 2241 | PJSON_INCREMENT_STAT(m_num_numeric_chars); c = *++pStr; 2242 | } 2243 | } 2244 | 2245 | static const float s_neg[2] = { 1.0f, -1.0f }; 2246 | double v = f * s_neg[is_neg]; 2247 | int64 final_scale = scale + escale * escalesign; 2248 | if (static_cast(final_scale + 31) <= 62) 2249 | v *= globals::s_pow10_table[static_cast(final_scale) + 31]; 2250 | else 2251 | { 2252 | if ((final_scale < INT32_MIN) || (final_scale > INT32_MAX)) 2253 | return set_error(pStr, "Failed parsing numeric value"); 2254 | v *= pow(10.0, static_cast(final_scale)); 2255 | } 2256 | 2257 | pChild_variant->m_type = cJSONValueTypeDouble; 2258 | pChild_variant->m_data.m_flVal = v; 2259 | } 2260 | } 2261 | 2262 | break; 2263 | } 2264 | case '\0': 2265 | return set_error(pStr, "Premature end of string (expected name or value)"); 2266 | default: 2267 | return set_error(pStr, "Unrecognized character"); 2268 | } 2269 | } 2270 | } 2271 | 2272 | bool deserialize_start(uint8* pStr) 2273 | { 2274 | set_to_null(); 2275 | 2276 | #if PJSON_PARSE_STATS 2277 | m_parse_stats.clear(); 2278 | #endif 2279 | 2280 | m_allocator.reset(); 2281 | 2282 | m_pStart = pStr; 2283 | 2284 | m_pStr = skip_whitespace(pStr); 2285 | 2286 | if (!*m_pStr) 2287 | return set_error(m_pStr, "Nothing to deserialize"); 2288 | 2289 | bool success = false; 2290 | 2291 | uint8 c = *m_pStr; 2292 | if ((c == '{') || (c == '[')) 2293 | { 2294 | set_to_node(c == '{'); PJSON_INCREMENT_STAT(m_num_control); 2295 | success = deserialize_internal(); 2296 | } 2297 | else 2298 | return set_error(m_pStr, "Root value must be an object or array"); 2299 | 2300 | if (success) 2301 | { 2302 | m_pStr = skip_whitespace(m_pStr); 2303 | success = !*m_pStr; 2304 | if (!success) 2305 | set_error(m_pStr, "Unknown data at end of document"); 2306 | } 2307 | 2308 | return success; 2309 | } 2310 | }; 2311 | 2312 | } // namespace pjson 2313 | 2314 | #endif // PJSON_H --------------------------------------------------------------------------------