├── LICENSE.md
├── README.md
├── admin
├── config.php
├── inc.admin.php
├── inc.authorize.php
└── server-status.php
├── doc
├── ADVANCED.md
├── GENESYS.md
├── HEALTH_CHECK.md
├── INSTALL.md
├── genesys.png
├── hp.png
├── logo_dark.svg
├── logo_light.svg
└── server-status-solproxy.png
├── gosol
├── go.mod
├── go.sum
├── handle_kvstore
│ ├── backend.go
│ └── kvstore.go
├── main
│ ├── conf.json
│ ├── main.go
│ └── server-status.html
├── passthrough
│ └── handle_passthrough.go
├── plugins
│ ├── common
│ │ └── common.go
│ ├── genesys
│ │ ├── genesys.go
│ │ └── get_token.go
│ └── plugin_manager.go
├── solana
│ ├── handle_solana_01
│ │ └── handle_solana_01.go
│ ├── handle_solana_admin
│ │ ├── handle_solana_admin.go
│ │ ├── parse_header.go
│ │ └── sol_config_reader.go
│ ├── handle_solana_info
│ │ └── handle_solana_info.go
│ └── handle_solana_raw
│ │ ├── public_passthrough_proxy.go
│ │ └── solana_raw.go
└── solana_proxy
│ ├── client
│ ├── client.go
│ ├── last_error.go
│ ├── maintenance.go
│ ├── request.go
│ ├── request_basic.go
│ ├── stats.go
│ ├── status.go
│ ├── status
│ │ └── status.go
│ └── throttle
│ │ ├── group.go
│ │ ├── read_config.go
│ │ ├── stats.go
│ │ ├── status.go
│ │ └── throttle.go
│ ├── custom_health_checker.go
│ ├── manager.go
│ ├── scheduler.go
│ ├── scheduler_pick.go
│ ├── status.go
│ └── throttle
│ └── throttle.go
└── handler_socket2
├── byteslabs
├── byteslabs.go
├── byteslabs_test.go
└── status.go
├── byteslabs2
├── byteslabs.go
├── byteslabs_test.go
└── status.go
├── compress
├── engine.go
├── flate.go
├── simple.go
├── snappy.go
├── snappy
│ ├── .gitignore
│ ├── AUTHORS
│ ├── CONTRIBUTORS
│ ├── LICENSE
│ ├── README
│ ├── cmd
│ │ └── snappytool
│ │ │ └── main.go
│ ├── decode.go
│ ├── decode_amd64.go
│ ├── decode_amd64.s
│ ├── decode_other.go
│ ├── encode.go
│ ├── encode_amd64.go
│ ├── encode_amd64.s
│ ├── encode_other.go
│ ├── golden_test.go
│ ├── gox.mod
│ ├── misc
│ │ └── main.cpp
│ ├── snappy.go
│ ├── snappy_test.go
│ └── testdata
│ │ ├── Mark.Twain-Tom.Sawyer.txt
│ │ └── Mark.Twain-Tom.Sawyer.txt.rawsnappy
└── status.go
├── compression_ex.go
├── config
├── config.go
├── config_loader.go
└── subattr.go
├── conn_ex.go
├── go.mod
├── go.sum
├── h_s.__go
├── handle_echo
└── echo.go
├── handle_profiler
└── profiler.go
├── handler_socket.go
├── hs_engine.go
├── hs_http.go
├── hs_params.go
├── hs_udp.go
├── hs_udp_v1.go
├── hscommon
├── hscommon.go
├── hscommon.go.bak
└── str.go
├── oslimits
├── oslimits_freebsd.go
├── oslimits_linux.go
└── oslimits_windows.go
└── stats
├── stats.go
├── stats_history.go
└── status.go
/LICENSE.md:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2022 HowRare.is
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
5 |
6 | # HowRare Solana RPC proxy
7 | HowRare Solana RPC proxy is there to allow project creators to freely route Solana RPC calls to different Solana nodes utilizing prioritization and capping.
8 |
9 | It allows to:
10 | - Route requests between fast local node(s) with partial chain data and remote node(s) with full chain history
11 | - Keep requests below allocated limits with per-function, per-request and per-transfer capping
12 | - Spread the load across many nodes/providers
13 | - Automatically detect and skip failed/overloaded/timeouting nodes
14 | - Automatically re-do failed requests on different node if they timed-out or returned an error
15 |
16 | What's new in Solproxy 0.3?
17 | - Removed genesysgo support
18 | - Authorization. You can add authorization or any other header using "header" parameter. Separate individual headers using newline.
19 | ```json
20 | "SOL_NODES":[{"header":"Authorization:Bearer b6082xxxxx\nCache-Control:no-cache", "url":"https://rpc.hellomoon.io/", "public":false, "score_modifier":-90000}],
21 | ```
22 |
23 | What's new in Solproxy 0.2?
24 | - GenesysGo support, see [adding GenesysGo node](doc/GENESYS.md)
25 | - Health check for pausing lagging nodes, see [setup instructions](doc/HEALTH_CHECK.md)
26 | - Updated dashboard UI
27 | - Allow to throttle using any time interval (eg. cap requests daily or per second)
28 |
29 |
30 |
32 |
33 | ## Building the software
34 | Run following commands to build for windows / linux. Golang 1.18 required.
35 | cd solproxy/gosol/main
36 | go build main.go
37 |
38 |
39 | Now you can run main or see [Installation Instructions](doc/INSTALL.md)
40 |
41 | ## Node types
42 | There are 2 node types defined
43 | - Public - this node stores full archival chain data
44 | - Private - fast local node (usually with partial chain data)
45 | If you don't need to distinct and you want to use the proxy just to route your requests to different providers for loadbalancing / failover - you should setup all nodes as a private type.
46 |
47 | This should be default, simplest mode of operation. You'll setup all your nodes as private nodes, and then you can connect to Solana proxy via any api just like you'd do to a "normal" Solana node, using port 7778.
48 |
49 | ## Configuration
50 | ```json
51 | {
52 | "BIND_TO": "h127.0.0.1:7778,h8.8.8.8:7778,",
53 |
54 | "FORCE_START":true,
55 | "DEBUG":false,
56 | "VERBOSE":false,
57 | "RUN_SERVICES":"*",
58 |
59 | "SOL_NODES":[{"url":"http://127.0.0.1:8899", "public":false, "score_modifier":-90000},
60 | {"url":"https://api.mainnet-beta.solana.com", "public":false, "throttle":"r,80,10;f,30,10;d,80000000,30", "probe_time":20},
61 | {"url":"https://solana-api.projectserum.com", "public":false, "throttle":"r,80,10;f,30,10;d,80000000,30", "probe_time":20}],
62 | }
63 | ```
64 | Configuration should be self-explanatory. You need to add h prefix before each IP the proxy will bind to. It'll listen for new connection on this IP/Port. There's a possibility to communicate with proxy using pure TCP by skipping the prefix.
65 |
66 | Throttle can be configured in following way:
67 | - r[equests],time_in_seconds,limit
68 | - f[unction call],time_in_seconds,limit
69 | - d[ata received],time_in_seconds,limit in bytes
70 |
71 | ## Accessing proxy information
72 | http://127.0.0.1:7778/?action=server-status
73 | You can access server-status page by using server-status action. There's also PHP script available to password-protect the status page so it can be accessible from outside.
74 |
75 | http://127.0.0.1:7778/?action=getSolanaInfo
76 | This url will return throttling status for public and private nodes.
77 |
78 | http://127.0.0.1:7778/?action=getFirstAvailableBlock
79 | Gets first available block for public and private nodes.
80 |
81 | ## Throttling
82 | There is automatic throttling/routing implemented. If node is throttled the request will be routed to different node. If all available nodes are throttled so there's no node to pick to run the request - you will get response with error attribute and issue description.
83 | ```json
84 | {"error":"Throttled public node, please wait","throttle_info":{"requests":{"description":"requests made","max":99,"value":3},"requests_fn":{"description":"requests made calling single function","max":39,"value":3},"received":{"description":"bytes received","max":1000000,"value":4735645}},"throttle_timespan_seconds":12,"throttled":true,"throttled_comment":"Too much data received 4735645/1000000"}
85 | ```
86 |
87 |
88 | Please see [Advanced usage](doc/ADVANCED.md) for information about more complex usage scenarios.
89 |
--------------------------------------------------------------------------------
/admin/config.php:
--------------------------------------------------------------------------------
1 | "change_password_here"]);
--------------------------------------------------------------------------------
/admin/inc.admin.php:
--------------------------------------------------------------------------------
1 | 'Unknown error'];
15 | }
16 | if (isset($data['error']))
17 | self::$last_error = $data['error'];
18 | return $data;
19 | }
20 |
21 | static function nodeList() {
22 | self::$last_error = false;
23 | return self::run(self::$host."?action=solana_admin");
24 | }
25 |
26 | static function nodeAdd($config_json, $replace_node_id = false)
27 | {
28 | self::$last_error = false;
29 | $config_json = urlencode($config_json);
30 | return self::run(self::$host."?action=solana_admin_add&node={$config_json}&remove_id={$replace_node_id}");
31 | }
32 |
33 | static function nodeRemove($node_id = false)
34 | {
35 | self::$last_error = false;
36 | $data = self::run(self::$host."?action=solana_admin_remove&id={$node_id}");
37 | return $data;
38 | }
39 | }
40 |
41 | /*
42 | echo "";
43 | print_r(admin::nodeList());
44 | echo "
12 | ... ,"plugin-genesys": {"client_id":"c26fe6*","pk": "4rZGkEjJ8qcF*"}, ...
13 |
14 |
15 | After placing your data in conf.json you need to start solproxy. If everything's right you should see following message, which will contain updated configuration you should use for production, to not store your private key.
16 |
17 |
18 | Warning: You should copy genesys plugin config from below for production, to not store unencrypted PK
19 | ------------------------------------------------------------------
20 | {"client_id":"c26fe6*","msg":"3rYuxoyygKo*",
21 | "public_key":"2PQuSo*"}
22 | ------------------------------------------------------------------
23 |
24 |
25 | Notes:
26 | - Private key allows anyone to access your funds, so you should enter your configuration **only locally**, then on production you should use the configuration settings generated by solproxy.
27 | - For additional security never use your main wallet, always create a new, empty one
28 |
29 |
30 |
32 |
33 | ## Adding node
34 | After you have authentication running you should be able to add your GenesysGo node normally. Authentication settings will be matched to GenesysGo node by your client_id
35 |
--------------------------------------------------------------------------------
/doc/HEALTH_CHECK.md:
--------------------------------------------------------------------------------
1 | # Healtch check plugin
2 | Healtch check is there so solproxy can automatically pause lagging nodes, using data which solproxy collects as heartbeat process.
3 |
4 |
5 | ... "CUSTOM_HEALTH_CHECKER":{"run_every": 20, "max_block_lag": 1000, "max_data_age_ms": 30000} ...
6 |
7 |
8 | The process will run every **run_every seconds** and compare latest-block-number of all defined clients. If latest-block-number was acquired too long ago (> **max_data_age_ms**) the proxy will try to refresh it before doing comparison. Maximum latest-block-number from the cluster is determined, and lastly if any node's lag is greater than **max_block_lag** it'll be paused.
9 |
10 | Paused node will be displaying in gray on the dashboard and no user requests will be forwarded to that node, untill it'll catch up with the rest of the network.
11 |
12 |
13 |
15 |
16 | As you can see on the picture, latest-block-number data was too old for node #3 (>30s), so it was re-fetched. It's good to run these checks infrequently and set proper margins, to conserve requests.
17 |
18 | You can configure solana public nodes as sources of latest-block-number data by using priority, so user requests won't be sent to them.
--------------------------------------------------------------------------------
/doc/INSTALL.md:
--------------------------------------------------------------------------------
1 | ## Installation
2 | It's not required to install, you can run directly from console and the proxy will work right after compiling, using standard Solana public nodes.
3 |
4 | ## Installing as a service
5 | - Create a new user, it can be called solproxy
6 | - Copy the content of gosol/main into /home/solproxy (it needs to contain compiled sources) or just download a package from this website and put everything under /home/solproxy
7 | - Create a directory /home/solproxy/log owned by solproxy user
8 | - Run sudo vi /etc/systemd/system/solproxy.service and place the following content into solproxy.service file
9 |
10 | [Unit]
11 | After=network-online.target
12 | Wants=network-online.target
13 | Description=Solana Proxy Service
14 |
15 | [Service]
16 | User=solproxy
17 | LimitNOFILE=524288
18 | LimitMEMLOCK=1073741824
19 | LimitNICE=-10
20 | Nice=-10
21 | ExecStart=/bin/sh -c 'cd /home/solproxy; export GODEBUG=gctrace=1; started=`date --rfc-3339=seconds`; echo Starting Solana Proxy $started; ./main 1>"log/log-$started.txt" 2>"log/error-$started.log.txt";'
22 | Type=simple
23 | PrivateNetwork=false
24 | PrivateTmp=false
25 | ProtectSystem=false
26 | ProtectHome=false
27 | KillMode=control-group
28 | Restart=always
29 | DefaultTasksMax=65536
30 | TasksMax=65536
31 | RestartSec=30
32 | StartLimitIntervalSec=200
33 | StartLimitBurst=10
34 |
35 | [Install]
36 | WantedBy=multi-user.target
37 | - Run systemctl daemon-reload
38 | - Run systemctl enable worker-node
39 |
40 | The proxy should be now running
41 |
--------------------------------------------------------------------------------
/doc/genesys.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HowRareIs/solproxy/aec513a3c74cc390704f0a78a26a23e8bd06f318/doc/genesys.png
--------------------------------------------------------------------------------
/doc/hp.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HowRareIs/solproxy/aec513a3c74cc390704f0a78a26a23e8bd06f318/doc/hp.png
--------------------------------------------------------------------------------
/doc/logo_dark.svg:
--------------------------------------------------------------------------------
1 | " + ret + " "
26 | })
27 | }
28 |
29 | func (this *Handle_kvstore) Info() string {
30 | return "Basic KV storage plugin"
31 | }
32 |
33 | func (this *Handle_kvstore) GetActions() []string {
34 | return []string{"keyGet", "keySet"}
35 | }
36 |
37 | func (this *Handle_kvstore) HandleAction(action string, data *handler_socket2.HSParams) string {
38 |
39 | k := data.GetParam("k", "")
40 | if len(k) == 0 {
41 | ret := "?k=key name (required)." + ret + " "
70 | })
71 | }
72 |
--------------------------------------------------------------------------------
/gosol/solana/handle_solana_01/handle_solana_01.go:
--------------------------------------------------------------------------------
1 | package handle_solana_01
2 |
3 | import (
4 | "gosol/solana_proxy"
5 | "gosol/solana_proxy/client"
6 | "strings"
7 |
8 | "github.com/slawomir-pryczek/HSServer/handler_socket2"
9 | )
10 |
11 | type Handle_solana_01 struct {
12 | }
13 |
14 | func (this *Handle_solana_01) Initialize() {
15 | }
16 |
17 | func (this *Handle_solana_01) Info() string {
18 | return "This plugin will return minimum block numbers for all nodes"
19 | }
20 |
21 | func (this *Handle_solana_01) GetActions() []string {
22 | return []string{"getBlock", "getTransaction", "getBalance", "getTokenSupply"}
23 | }
24 |
25 | func (this *Handle_solana_01) HandleAction(action string, data *handler_socket2.HSParams) string {
26 |
27 | sch := solana_proxy.MakeScheduler()
28 | if data.GetParamI("public", 0) == 1 {
29 | sch.ForcePublic(true)
30 | }
31 | if data.GetParamI("private", 0) == 1 {
32 | sch.ForcePrivate(true)
33 | }
34 | cl := sch.GetAnyClient()
35 | if cl == nil {
36 | return `{"error":"can't find appropriate client"}`
37 | }
38 |
39 | mstv_i := 0
40 | if action == "getBlock" || action == "getTransaction" {
41 | mstv := data.GetParam("maxSupportedTransactionVersion", "")
42 | if len(mstv) > 0 {
43 | mstv_i = -2
44 | if strings.EqualFold("legacy", mstv) {
45 | mstv_i = -1
46 | }
47 | if mstv == "0" {
48 | mstv_i = 0
49 | }
50 | }
51 | if mstv_i == -2 {
52 | return `{"error":"maxSupportedTransactionVersion needs to be 0 or legacy"}`
53 | }
54 | }
55 |
56 | if action == "getBlock" {
57 | block_no := data.GetParamI("block", -1)
58 | if block_no == -1 {
59 | return `{"error":"provide block number as &block=123"}`
60 | }
61 |
62 | sch.SetMinBlock(block_no)
63 | ret, result := cl.GetBlock(block_no, mstv_i)
64 |
65 | for result != client.R_OK {
66 | cl = sch.GetPublicClient()
67 | if cl == nil {
68 | cl = sch.GetAnyClient()
69 | }
70 | if cl == nil {
71 | return `{"error":"can't find appropriate client (2)"}`
72 | }
73 | ret, result = cl.GetBlock(block_no, mstv_i)
74 | }
75 | data.FastReturnBNocopy(ret)
76 | return ""
77 | }
78 |
79 | if action == "getTransaction" {
80 | hash := data.GetParam("hash", "")
81 | if len(hash) == 0 {
82 | return `{"error":"provide transaction &hash=123"}`
83 | }
84 |
85 | ret, result := cl.GetTransaction(hash, mstv_i)
86 | for result != client.R_OK {
87 | cl = sch.GetPublicClient()
88 | if cl == nil {
89 | cl = sch.GetAnyClient()
90 | }
91 | if cl == nil {
92 | return `{"error":"can't find appropriate client (2)"}`
93 | }
94 |
95 | ret, result = cl.GetTransaction(hash, mstv_i)
96 | }
97 |
98 | data.FastReturnBNocopy(ret)
99 | return ""
100 | }
101 |
102 | if action == "getBalance" || action == "getTokenSupply" {
103 | pubkey := data.GetParam("pubkey", "")
104 | if len(pubkey) == 0 {
105 | return `{"error":"provide pubkey &pubkey=123, and optionally &commitment="}`
106 | }
107 | commitment := data.GetParam("commitment", "")
108 |
109 | ret, result := cl.SimpleCall(action, pubkey, commitment)
110 | for result != client.R_OK {
111 | cl = sch.GetPublicClient()
112 | if cl == nil {
113 | cl = sch.GetAnyClient()
114 | }
115 | if cl == nil {
116 | return `{"error":"can't find appropriate client (2)"}`
117 | }
118 | ret, result = cl.SimpleCall(action, pubkey, commitment)
119 | }
120 | data.FastReturnBNocopy(ret)
121 | return ""
122 | }
123 |
124 | return "No function?!"
125 | }
126 |
--------------------------------------------------------------------------------
/gosol/solana/handle_solana_admin/handle_solana_admin.go:
--------------------------------------------------------------------------------
1 | package handle_solana_admin
2 |
3 | import (
4 | "bytes"
5 | "encoding/json"
6 | "fmt"
7 | "github.com/slawomir-pryczek/HSServer/handler_socket2"
8 | "gosol/solana_proxy"
9 | "math"
10 | )
11 |
12 | type Handle_solana_admin struct {
13 | }
14 |
15 | func (this *Handle_solana_admin) Initialize() {
16 | }
17 |
18 | func (this *Handle_solana_admin) Info() string {
19 | return "This plugin will allow to do proxy administration"
20 | }
21 |
22 | func (this *Handle_solana_admin) GetActions() []string {
23 | return []string{"solana_admin", "solana_admin_remove", "solana_admin_add"}
24 | }
25 |
26 | func (this *Handle_solana_admin) HandleAction(action string, data *handler_socket2.HSParams) string {
27 |
28 | err := func(s string) string {
29 | ret := make(map[string]interface{})
30 | ret["error"] = s
31 | tmp, _ := json.Marshal(ret)
32 | return string(tmp)
33 | }
34 |
35 | ok := func(s interface{}) string {
36 | ret := make(map[string]interface{})
37 | ret["result"] = s
38 | tmp, _ := json.Marshal(ret)
39 | return string(tmp)
40 | }
41 |
42 | if action == "solana_admin" {
43 | sch := solana_proxy.MakeScheduler()
44 | clients := sch.GetAll(true, true)
45 | clients = append(clients, sch.GetAll(false, true)...)
46 |
47 | out := make(map[string]interface{}, 0)
48 | for _, client := range clients {
49 | _tmp := client.GetInfo()
50 | out[fmt.Sprintf("client_#%d", _tmp.ID)] = _tmp
51 | }
52 |
53 | _tmp, _ := json.Marshal(out)
54 | data.FastReturnBNocopy(_tmp)
55 | return ""
56 | }
57 |
58 | if action == "solana_admin_remove" {
59 | id := data.GetParamI("id", -1)
60 | if id < 0 {
61 | return "Please provide client's &id="
62 | }
63 |
64 | if solana_proxy.ClientRemove(uint64(id)) {
65 | return ok(fmt.Sprintf("Removed client id: %d", id))
66 | } else {
67 | return err("Can't find client, nothing done")
68 | }
69 | }
70 |
71 | if action == "solana_admin_add" {
72 |
73 | id := data.GetParamI("remove_id", -1)
74 | node_id := uint64(0)
75 | if id < 0 {
76 | node_id = math.MaxUint64
77 | } else {
78 | node_id = uint64(id)
79 | }
80 | node := data.GetParam("node", "")
81 | if len(node) == 0 {
82 | return "Please provide &node={...JSON...} as node config, additionally you can provide &remove=node_id to replace the node with new one"
83 | }
84 |
85 | var cfg_tmp map[string]interface{}
86 | d := json.NewDecoder(bytes.NewReader([]byte(node)))
87 | d.UseNumber()
88 | if _err := d.Decode(&cfg_tmp); _err != nil {
89 | return err(_err.Error())
90 | }
91 |
92 | new_node := NodeRegisterFromConfig(cfg_tmp)
93 | if new_node == nil {
94 | return err("Error creating new node, something went wrong. Please check URL and config")
95 | }
96 | solana_proxy.ClientRemove(node_id)
97 | return ok(new_node.GetInfo())
98 | }
99 |
100 | return err("Something went wrong in admin module")
101 | }
102 |
--------------------------------------------------------------------------------
/gosol/solana/handle_solana_admin/parse_header.go:
--------------------------------------------------------------------------------
1 | package handle_solana_admin
2 |
3 | import (
4 | "net/http"
5 | "regexp"
6 | "strings"
7 | )
8 |
9 | var rxNewline = regexp.MustCompile(`[\r\n]+`)
10 |
11 | func parseHeader(s string) http.Header {
12 | h := make(map[string][]string)
13 |
14 | for _, v := range strings.Split(s, "\n") {
15 | if strings.Index(v, ":") == -1 {
16 | continue
17 | }
18 | v = rxNewline.ReplaceAllString(v, "")
19 |
20 | tmp := strings.Split(v, ":")
21 | if len(tmp) < 2 {
22 | continue
23 | }
24 | tmp[0] = http.CanonicalHeaderKey(strings.Trim(tmp[0], "\r\n\t "))
25 | tmp[1] = strings.Trim(tmp[1], "\r\n\t ")
26 | if len(tmp[0]) == 0 || len(tmp[1]) == 0 {
27 | continue
28 | }
29 | h[tmp[0]] = []string{tmp[1]}
30 | }
31 | if len(h) == 0 {
32 | return nil
33 | }
34 | return http.Header(h)
35 | }
36 |
--------------------------------------------------------------------------------
/gosol/solana/handle_solana_admin/sol_config_reader.go:
--------------------------------------------------------------------------------
1 | package handle_solana_admin
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "gosol/solana_proxy"
7 | "gosol/solana_proxy/client"
8 | "gosol/solana_proxy/client/throttle"
9 | "math"
10 | "net/http"
11 | "reflect"
12 | "strings"
13 | )
14 |
15 | func NodeRegister(endpoint string, header http.Header, public bool, probe_time int, throttle []*throttle.Throttle) *client.SOLClient {
16 | if len(endpoint) == 0 {
17 | return nil
18 | }
19 | max_conn := 50
20 | if public {
21 | max_conn = 10
22 | }
23 | endpoint = strings.Trim(endpoint, "\r\n\t ")
24 |
25 | if probe_time == -1 {
26 | if public {
27 | probe_time = 10
28 | } else {
29 | probe_time = 1
30 | }
31 | }
32 |
33 | cl := client.MakeClient(endpoint, header, public, probe_time, max_conn, throttle)
34 | solana_proxy.ClientManage(cl, math.MaxUint64)
35 | return cl
36 | }
37 |
38 | func _get_cfg_data[T any](node map[string]interface{}, attr string, def T) T {
39 | if val, ok := node[attr]; ok {
40 | switch val.(type) {
41 | case T:
42 | return val.(T)
43 | default:
44 | fmt.Println("Warning: type mismatch for", attr, "attribute is", reflect.TypeOf(val).Name(), ", needs to be ", reflect.TypeOf(new(T)).Name())
45 | }
46 | }
47 | return def
48 | }
49 |
50 | func NodeRegisterFromConfig(node map[string]interface{}) *client.SOLClient {
51 |
52 | url := _get_cfg_data(node, "url", "")
53 | public := _get_cfg_data(node, "public", false)
54 | score_modifier, _ := _get_cfg_data(node, "score_modifier", json.Number("0")).Int64()
55 | probe_time, _ := _get_cfg_data(node, "probe_time", json.Number("-1")).Int64()
56 | header := parseHeader(_get_cfg_data(node, "header", ""))
57 |
58 | if url == "" {
59 | fmt.Println("Cannot read node config (no url) ... skipping")
60 | return nil
61 | }
62 |
63 | thr := ([]*throttle.Throttle)(nil)
64 | logs := []string{}
65 | fmt.Printf("## Node: %s Public: %v, score modifier: %d\n", url, public, score_modifier)
66 |
67 | if val, ok := node["throttle"]; ok {
68 | switch val.(type) {
69 | case string:
70 | thr, logs = throttle.MakeFromConfig(val.(string))
71 | default:
72 | fmt.Println("Warning: Cannot read throttle settings, skipping throttling")
73 | }
74 | } else {
75 | if public {
76 | thr, logs = throttle.MakeForPublic()
77 | }
78 | }
79 |
80 | if thr == nil {
81 | thr = make([]*throttle.Throttle, 0, 1)
82 | thr = append(thr, throttle.Make())
83 | logs = append(logs, "Throttling disabled")
84 | }
85 | throttle.ThrottleGoup(thr).SetScoreModifier(int(score_modifier))
86 |
87 | for _, log := range logs {
88 | fmt.Println(" ", log)
89 | }
90 |
91 | return NodeRegister(url, header, public, int(probe_time), thr)
92 | }
93 |
--------------------------------------------------------------------------------
/gosol/solana/handle_solana_info/handle_solana_info.go:
--------------------------------------------------------------------------------
1 | package handle_solana_info
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | "gosol/solana_proxy"
7 |
8 | "github.com/slawomir-pryczek/HSServer/handler_socket2"
9 | )
10 |
11 | type Handle_solana_info struct {
12 | }
13 |
14 | func (this *Handle_solana_info) Initialize() {
15 | }
16 |
17 | func (this *Handle_solana_info) Info() string {
18 | return "This plugin will return solana nodes information"
19 | }
20 |
21 | func (this *Handle_solana_info) GetActions() []string {
22 | return []string{"getFirstAvailableBlock", "getSolanaInfo"}
23 | }
24 |
25 | func (this *Handle_solana_info) HandleAction(action string, data *handler_socket2.HSParams) string {
26 |
27 | _round := func(n float64) float64 {
28 | tmp := int(n * 1000.0)
29 | return float64(tmp/100) / 10.0
30 | }
31 |
32 | if action == "getSolanaInfo" {
33 | pub, priv, pub_max, priv_max := solana_proxy.GetMinMaxBlocks()
34 |
35 | ret := map[string]interface{}{}
36 | ret["first_available_block"] = map[string]string{
37 | "public": fmt.Sprintf("%d", pub),
38 | "private": fmt.Sprintf("%d", priv)}
39 |
40 | ret["last_available_block"] = map[string]string{
41 | "public": fmt.Sprintf("%d", pub_max),
42 | "private": fmt.Sprintf("%d", priv_max)}
43 |
44 | sch := solana_proxy.MakeScheduler()
45 | if data.GetParamI("public", 0) == 1 {
46 | sch.ForcePublic(true)
47 | }
48 | if data.GetParamI("private", 0) == 1 {
49 | sch.ForcePrivate(true)
50 | }
51 |
52 | // calculate limits
53 | var limits_pub_left = [5]int{0, 0, 0, 0, 0}
54 | var limits_priv_left = [5]int{0, 0, 0, 0, 0}
55 | tmp := [4]int{}
56 | for num, v := range sch.GetAll(true, true) {
57 | tmp[0], tmp[1], tmp[2], tmp[3] = v.GetThrottleLimitsLeft()
58 | for i := 0; i < 4; i++ {
59 | limits_pub_left[i] += tmp[i]
60 | }
61 | limits_pub_left[4]++
62 | ret[fmt.Sprintf("pub-%d", num)] = tmp
63 | }
64 | for num, v := range sch.GetAll(false, true) {
65 | tmp[0], tmp[1], tmp[2], tmp[3] = v.GetThrottleLimitsLeft()
66 | for i := 0; i < 4; i++ {
67 | limits_priv_left[i] += tmp[i]
68 | }
69 | limits_priv_left[4]++
70 | ret[fmt.Sprintf("priv-%d", num)] = tmp
71 | }
72 |
73 | // generate JSON
74 | _gen := func(data [5]int) map[string]interface{} {
75 | ret := map[string]interface{}{}
76 | ret["requests_left"] = data[0]
77 | ret["requests_single_left"] = data[1]
78 | ret["byte_received_left"] = data[2]
79 | if data[4] == 0 {
80 | ret["utilization_percent"] = 0
81 | } else {
82 | ret["utilization_percent"] = _round((float64(data[3]) / float64(data[4])) / 100.0)
83 | }
84 | ret["node_count"] = data[4]
85 | return ret
86 | }
87 | ret["public"] = _gen(limits_pub_left)
88 | ret["private"] = _gen(limits_priv_left)
89 | ret["comment"] = "Per node data is requests left / requests single left / bytes reveived left / utilization percentage"
90 | _tmp, _ := json.Marshal(ret)
91 | data.FastReturnBNocopy(_tmp)
92 | return ""
93 | }
94 |
95 | if action == "getFirstAvailableBlock" {
96 |
97 | pub, priv, _, _ := solana_proxy.GetMinMaxBlocks()
98 |
99 | ret := map[string]string{}
100 | ret["public"] = fmt.Sprintf("%d", pub)
101 | ret["private"] = fmt.Sprintf("%d", priv)
102 |
103 | _tmp, _ := json.Marshal(ret)
104 | data.FastReturnBNocopy(_tmp)
105 | return ""
106 | }
107 |
108 | return "No function ?!"
109 | }
110 |
--------------------------------------------------------------------------------
/gosol/solana/handle_solana_raw/public_passthrough_proxy.go:
--------------------------------------------------------------------------------
1 | package handle_solana_raw
2 |
3 | import (
4 | "encoding/json"
5 | "gosol/solana_proxy"
6 | "gosol/solana_proxy/client"
7 | "net/http"
8 | "strings"
9 |
10 | "github.com/slawomir-pryczek/HSServer/handler_socket2"
11 | )
12 |
13 | func _passthrough_err(err string) []byte {
14 | out := make(map[string]interface{}, 0)
15 | out["message"] = err
16 | out["code"] = 111
17 | out["proxy_error"] = true
18 | b, e := json.Marshal(out)
19 | if e != nil {
20 | b = []byte("\"Unknown error\"")
21 | }
22 | return []byte("{\"error\":" + string(b) + "}")
23 | }
24 |
25 | func init() {
26 |
27 | handler_socket2.HTTPPluginRegister(func(w http.ResponseWriter, header http.Header, get map[string]string, post []byte) bool {
28 |
29 | is_sol_rpc := strings.EqualFold("application/json", header.Get("Content-Type"))
30 | if !is_sol_rpc {
31 | return false
32 | }
33 |
34 | for i := 0; i < len(post); i++ {
35 | if post[i] == '{' {
36 | is_sol_rpc = true
37 | break
38 | }
39 | if post[i] == '\n' || post[i] == '\r' || post[i] == ' ' {
40 | continue
41 | }
42 | break // we couldn't find JSON bracket, so it's not SOL RPC
43 | }
44 | if !is_sol_rpc {
45 | return false
46 | }
47 |
48 | sch := solana_proxy.MakeScheduler()
49 | clients := sch.GetAllSorted(false, false)
50 | if len(clients) == 0 {
51 | w.Write(_passthrough_err("Can't find any client"))
52 | return true
53 | }
54 |
55 | // loop over workers, if we have "throttled" returned it'll try other workers
56 | errors := 0
57 | for _, cl := range clients {
58 | resp_type, resp_data := cl.RequestForward(post)
59 | if resp_type == client.R_OK {
60 | w.Write(resp_data)
61 | return true
62 | }
63 |
64 | if resp_type == client.R_ERROR {
65 | errors++
66 | if errors >= 2 {
67 | w.Write(_passthrough_err("Request failed (e)"))
68 | return true
69 | }
70 | }
71 | }
72 |
73 | w.Write(_passthrough_err("Request failed"))
74 | return true
75 | })
76 | }
77 |
--------------------------------------------------------------------------------
/gosol/solana/handle_solana_raw/solana_raw.go:
--------------------------------------------------------------------------------
1 | package handle_solana_raw
2 |
3 | import (
4 | "bytes"
5 | "encoding/json"
6 | "gosol/solana_proxy"
7 | "gosol/solana_proxy/client"
8 |
9 | "github.com/slawomir-pryczek/HSServer/handler_socket2"
10 | )
11 |
12 | type Handle_solana_raw struct {
13 | }
14 |
15 | func (this *Handle_solana_raw) Initialize() {
16 | }
17 |
18 | func (this *Handle_solana_raw) Info() string {
19 | return "This plugin will allow to issue raw solana requests"
20 | }
21 |
22 | func (this *Handle_solana_raw) GetActions() []string {
23 | return []string{"solanaRaw"}
24 | }
25 |
26 | func (this *Handle_solana_raw) HandleAction(action string, data *handler_socket2.HSParams) string {
27 |
28 | // get first client!
29 | sch := solana_proxy.MakeScheduler()
30 | if data.GetParamI("public", 0) == 1 {
31 | sch.ForcePublic(true)
32 | }
33 | if data.GetParamI("private", 0) == 1 {
34 | sch.ForcePrivate(true)
35 | }
36 | cl := sch.GetAnyClient()
37 | if cl == nil {
38 | return `{"error":"can't find appropriate client"}`
39 | }
40 |
41 | // run the request
42 | is_req_ok := func(data []byte) bool {
43 | v := make(map[string]interface{})
44 | dec := json.NewDecoder(bytes.NewReader(data))
45 | dec.UseNumber()
46 | dec.Decode(&v)
47 |
48 | switch v["result"].(type) {
49 | case nil:
50 | return false
51 | }
52 | return true
53 | }
54 |
55 | method := data.GetParam("method", "")
56 | params := data.GetParam("params", "")
57 | if len(method) == 0 {
58 | return `{"error":"provide transaction &method=getConfirmedBlock and optionally ¶ms=[94435095] add &public=1 if you want to force the request to be run on public node"}`
59 | }
60 |
61 | // Try first client (private by default)
62 | ret, result := cl.RequestBasic(method, params)
63 | if ret != nil && result == client.R_OK && is_req_ok(ret) {
64 | data.FastReturnBNocopy(ret)
65 | return ""
66 | }
67 |
68 | // Try public client, if private failed
69 | cl = sch.GetPublicClient()
70 | if cl != nil {
71 | ret, result = cl.RequestBasic(method, params)
72 | }
73 | if ret != nil && result == client.R_OK && is_req_ok(ret) {
74 | data.FastReturnBNocopy(ret)
75 | return ""
76 | }
77 |
78 | // last result, try anything which is not throttled!
79 | for result == client.R_THROTTLED {
80 | cl = sch.GetAnyClient()
81 | if cl == nil {
82 | break
83 | }
84 | ret, result = cl.RequestBasic(method, params)
85 | }
86 | if ret != nil && result == client.R_OK && is_req_ok(ret) {
87 | data.FastReturnBNocopy(ret)
88 | return ""
89 | }
90 |
91 | // return error, if we were not able to process the request correctly
92 | if ret != nil {
93 | data.FastReturnBNocopy(ret)
94 | return ""
95 | }
96 | return `{"error":"unknown issue"}`
97 | }
98 |
--------------------------------------------------------------------------------
/gosol/solana_proxy/client/client.go:
--------------------------------------------------------------------------------
1 | package client
2 |
3 | import (
4 | "gosol/solana_proxy/client/throttle"
5 | "net/http"
6 | "sync"
7 | "sync/atomic"
8 | "time"
9 | )
10 |
11 | type SOLClientAttr int
12 |
13 | const (
14 | CLIENT_CONSERVE_REQUESTS SOLClientAttr = 1 << 0
15 | )
16 |
17 | func (this *SOLClient) SetAttr(attrs SOLClientAttr) {
18 | this.attr = attrs
19 | }
20 |
21 | func (this *SOLClient) SetPaused(paused bool, comment string) {
22 | this.mu.Lock()
23 | this.is_paused = paused
24 | if this.is_paused {
25 | this.is_paused_comment = comment
26 | }
27 | this.mu.Unlock()
28 | }
29 |
30 | type SOLClient struct {
31 | id uint64
32 | client *http.Client
33 | endpoint string
34 | header http.Header
35 | is_public_node bool
36 | available_block_first int
37 | available_block_first_ts int64
38 | available_block_last int
39 | available_block_last_ts int64
40 |
41 | is_disabled bool
42 | is_paused bool
43 | is_paused_comment string
44 |
45 | stat_running int
46 | stat_total stat
47 | stat_last_60 [60]stat
48 | stat_last_60_pos int
49 |
50 | version_major int
51 | version_minor int
52 | version string
53 | version_ts int64
54 |
55 | mu sync.Mutex
56 | serial_no uint64
57 |
58 | attr SOLClientAttr
59 | throttle []*throttle.Throttle
60 |
61 | _probe_time int
62 | _probe_log string
63 |
64 | _last_error LastError
65 | }
66 |
67 | type Solclientinfo struct {
68 | ID uint64
69 | Endpoint string
70 | Is_public_node bool
71 | Available_block_first int
72 | Available_block_first_ts int64
73 | Available_block_last int
74 | Available_block_last_ts int64
75 | Is_disabled bool
76 | Is_throttled bool
77 | Is_paused bool
78 |
79 | Attr SOLClientAttr
80 | Score int
81 | }
82 |
83 | func (this *SOLClient) GetEndpoint() string {
84 | this.mu.Lock()
85 | ret := this.endpoint
86 | this.mu.Unlock()
87 |
88 | return ret
89 | }
90 |
91 | func (this *SOLClient) GetInfo() *Solclientinfo {
92 |
93 | ret := Solclientinfo{}
94 |
95 | this.mu.Lock()
96 | ret.ID = this.id
97 | ret.Endpoint = this.endpoint
98 | ret.Is_public_node = this.is_public_node
99 | ret.Available_block_first = this.available_block_first
100 | ret.Available_block_first_ts = this.available_block_first_ts
101 | ret.Available_block_last = this.available_block_last
102 | ret.Available_block_last_ts = this.available_block_last_ts
103 | ret.Is_disabled = this.is_disabled
104 | ret.Is_paused = this.is_paused
105 |
106 | tmp := throttle.ThrottleGoup(this.throttle).GetThrottleScore()
107 | ret.Score = tmp.Score
108 | ret.Is_throttled = tmp.Throttled
109 |
110 | ret.Attr = this.attr
111 | this.mu.Unlock()
112 |
113 | return &ret
114 | }
115 |
116 | var new_client_id = uint64(0)
117 |
118 | func MakeClient(endpoint string, header http.Header, is_public_node bool, probe_time int, max_conns int, throttle []*throttle.Throttle) *SOLClient {
119 |
120 | tr := &http.Transport{
121 | MaxIdleConns: max_conns,
122 | MaxConnsPerHost: max_conns,
123 | IdleConnTimeout: 10 * time.Second,
124 | DisableCompression: true}
125 |
126 | ret := SOLClient{}
127 | ret.client = &http.Client{Transport: tr, Timeout: 5 * time.Second}
128 | ret.endpoint = endpoint
129 | ret.header = header
130 | ret.is_public_node = is_public_node
131 | ret._probe_time = probe_time
132 | ret.stat_total.stat_request_by_fn = make(map[string]int)
133 | for i := 0; i < len(ret.stat_last_60); i++ {
134 | ret.stat_last_60[i].stat_request_by_fn = make(map[string]int)
135 | }
136 |
137 | ret.throttle = throttle
138 | ret._maintenance()
139 |
140 | ret.id = atomic.AddUint64(&new_client_id, 1)
141 | return &ret
142 | }
143 |
144 | func (this *SOLClient) GetThrottleLimitsLeft() (int, int, int, int) {
145 | this.mu.Lock()
146 | defer this.mu.Unlock()
147 | return throttle.ThrottleGoup(this.throttle).GetLimitsLeft()
148 | }
149 |
--------------------------------------------------------------------------------
/gosol/solana_proxy/client/last_error.go:
--------------------------------------------------------------------------------
1 | package client
2 |
3 | import (
4 | "fmt"
5 | "io/ioutil"
6 | "net/http"
7 | "strings"
8 | "time"
9 | )
10 |
11 | type LastError struct {
12 | str string
13 | details string
14 |
15 | call string
16 | call_ts int64
17 | counter int
18 | }
19 |
20 | func isHTTPError(resp *http.Response, err error, post []byte) *LastError {
21 | if resp != nil && resp.StatusCode == 200 && err == nil {
22 | return nil
23 | }
24 |
25 | this := LastError{counter: -1}
26 | this.call = string(post)
27 | this.call_ts = time.Now().UnixNano() / 1000
28 |
29 | tmp := make([]string, 0, 10)
30 | if resp == nil {
31 | tmp = append(tmp, "No response from host")
32 | }
33 | if err != nil {
34 | tmp = append(tmp, "Error: "+err.Error())
35 | }
36 |
37 | if resp != nil && resp.StatusCode != 200 {
38 | tmp = append(tmp, "HTTP: "+resp.Status)
39 | }
40 | this.str = strings.Join(tmp, "\n")
41 |
42 | tmp = tmp[:0]
43 | if resp != nil && len(resp.Header) > 0 {
44 | tmp = append(tmp, "Response Headers:")
45 | for k, v := range resp.Header {
46 | tmp = append(tmp, k+": "+strings.Join(v, ", "))
47 | }
48 | tmp = append(tmp, "")
49 | }
50 |
51 | body := []byte(nil)
52 | if resp != nil && resp.Body != nil {
53 | body, _ = ioutil.ReadAll(resp.Body)
54 | }
55 | if body != nil {
56 | tmp = append(tmp, "Body:\n"+string(body))
57 | } else {
58 | tmp = append(tmp, "Body: -")
59 | }
60 |
61 | this.details = strings.Join(tmp, "\n")
62 | return &this
63 | }
64 |
65 | func isGenericError(err error, post []byte) *LastError {
66 | if err == nil {
67 | return nil
68 | }
69 |
70 | this := LastError{counter: -1}
71 | this.call = string(post)
72 | this.call_ts = time.Now().UnixNano() / 1000
73 |
74 | this.str = err.Error()
75 | this.details = "-"
76 | return &this
77 | }
78 |
79 | func (this LastError) String() string {
80 | header, details := this.Info()
81 | return header + "\n" + details
82 | }
83 |
84 | func (this LastError) Info() (string, string) {
85 | header := fmt.Sprintf("Error %d @%s", this.counter, time.UnixMicro(this.call_ts).Format("2006-01-02 15:04:05")) + " / " + this.str
86 | details := "Request Data:" + this.call + "\n\n" + this.details
87 | return header, details
88 | }
89 |
--------------------------------------------------------------------------------
/gosol/solana_proxy/client/maintenance.go:
--------------------------------------------------------------------------------
1 | package client
2 |
3 | import (
4 | "fmt"
5 | "gosol/solana_proxy/client/throttle"
6 | "time"
7 | )
8 |
9 | func (this *SOLClient) _maintenance() {
10 |
11 | _maint_stat := func(now int64) {
12 | this.mu.Lock()
13 |
14 | throttle.ThrottleGoup(this.throttle).OnMaintenance(int(now))
15 |
16 | _p := int(now % 60)
17 | this.stat_last_60_pos = _p
18 | this.stat_last_60[_p].stat_done = 0
19 | this.stat_last_60[_p].stat_error_json_decode = 0
20 | this.stat_last_60[_p].stat_error_json_marshal = 0
21 | this.stat_last_60[_p].stat_error_req = 0
22 | this.stat_last_60[_p].stat_error_resp = 0
23 | this.stat_last_60[_p].stat_error_resp_read = 0
24 | this.stat_last_60[_p].stat_ns_total = 0
25 |
26 | this.stat_last_60[_p].stat_request_by_fn = make(map[string]int)
27 | this.stat_last_60[_p].stat_bytes_received = 0
28 | this.stat_last_60[_p].stat_bytes_sent = 0
29 |
30 | _d, _req_ok, _req_err, _log := this._statsIsDead()
31 | this.is_disabled = _d
32 | this._probe_log = _log
33 |
34 | // if we don't have at least 1 requests,
35 | // run a request to check if the node is alive
36 | // this._probe_time related
37 | if _req_ok+_req_err < 1 && this._probe_time > 0 {
38 | go func() {
39 | this.GetVersion()
40 | }()
41 | }
42 | this.mu.Unlock()
43 | }
44 |
45 | _update_version := func() {
46 | _a, _b, _c, ok := this.GetVersion()
47 | if ok != R_OK {
48 | fmt.Println("Health: Can't get version for: ", this.endpoint)
49 | return
50 | }
51 | this.mu.Lock()
52 | this.version_major, this.version_minor, this.version = _a, _b, _c
53 | this.mu.Unlock()
54 | }
55 |
56 | _update_first_block := func() {
57 | _, _ok := this.GetFirstAvailableBlock()
58 | if _ok != R_OK {
59 | fmt.Println("Health: Can't get first block for: ", this.endpoint)
60 | return
61 | }
62 | }
63 |
64 | _update_last_block := func() {
65 | _, _ok := this.GetLastAvailableBlock()
66 | if _ok != R_OK {
67 | fmt.Println("Health: Can't get last block for: ", this.endpoint)
68 | return
69 | }
70 | }
71 |
72 | // run first update, get all data required for the node to work!
73 | _update_version()
74 | _update_first_block()
75 | _update_last_block()
76 | go func() {
77 | for {
78 | now := time.Now().Unix()
79 | time.Sleep(500 * time.Millisecond)
80 | _t := time.Now().Unix()
81 | if now >= _t {
82 | continue
83 | }
84 |
85 | // update version and first block
86 | now = _t
87 |
88 | // if we have probing time set - use that
89 | if pt := int64(this._probe_time); pt > 0 {
90 | pt_by3 := pt * 3
91 | if now%pt_by3 == 0 {
92 | _update_version()
93 | }
94 | if now%pt_by3 == pt {
95 | _update_first_block()
96 | }
97 | if now%pt_by3 == pt*2 {
98 | _update_last_block()
99 | }
100 | }
101 | }
102 | }()
103 |
104 | _maint_stat(time.Now().Unix())
105 | go func() {
106 | for {
107 | now := time.Now().Unix()
108 | time.Sleep(200 * time.Millisecond)
109 | _t := time.Now().Unix()
110 | if now >= _t {
111 | continue
112 | }
113 |
114 | now = _t
115 | _maint_stat(now)
116 | }
117 | }()
118 | }
119 |
--------------------------------------------------------------------------------
/gosol/solana_proxy/client/request.go:
--------------------------------------------------------------------------------
1 | package client
2 |
3 | /* This file adds support for requests on higher level, without error processing */
4 |
5 | import (
6 | "bytes"
7 | "fmt"
8 | "time"
9 |
10 | "encoding/json"
11 | "strconv"
12 | "strings"
13 | )
14 |
15 | func (this *SOLClient) _intcall(method string) (int, ResponseType) {
16 | ret, r_type := this.RequestBasic(method)
17 | if ret == nil {
18 | return 0, r_type
19 | }
20 |
21 | r := make(map[string]interface{})
22 | dec := json.NewDecoder(bytes.NewReader(ret))
23 | dec.UseNumber()
24 | dec.Decode(&r)
25 |
26 | switch v := r["result"].(type) {
27 | case json.Number:
28 | _ret, err := v.Int64()
29 | if err != nil {
30 | break
31 | }
32 | return int(_ret), r_type
33 | default:
34 | fmt.Println("Error in response for " + method + ": " + string(ret))
35 | }
36 | return 0, R_ERROR
37 | }
38 |
39 | func (this *SOLClient) GetFirstAvailableBlock() (int, ResponseType) {
40 | ret, r_type := this._intcall("getFirstAvailableBlock")
41 | if r_type == R_OK {
42 | _ts := time.Now().UnixMilli()
43 | this.mu.Lock()
44 | this.available_block_first = ret
45 | this.available_block_first_ts = _ts
46 | this.mu.Unlock()
47 | }
48 | return ret, r_type
49 | }
50 |
51 | func (this *SOLClient) GetLastAvailableBlock() (int, ResponseType) {
52 | ret, r_type := this._intcall("getBlockHeight")
53 | if r_type == R_OK {
54 | _ts := time.Now().UnixMilli()
55 | this.mu.Lock()
56 | this.available_block_last = ret
57 | this.available_block_last_ts = _ts
58 | this.mu.Unlock()
59 | }
60 | return ret, r_type
61 | }
62 |
63 | func (this *SOLClient) GetVersion() (int, int, string, ResponseType) {
64 |
65 | ret, r_type := this.RequestBasic("getVersion")
66 | if ret == nil {
67 | return 0, 0, "", r_type
68 | }
69 |
70 | type out_result struct {
71 | Solana_core string `json:"solana-core"`
72 | }
73 | type out_main struct {
74 | Jsonrpc string `json:"jsonrpc"`
75 | Result out_result `json:"result"`
76 | }
77 |
78 | tmp := &out_main{}
79 | json.Unmarshal(ret, tmp)
80 |
81 | if len(tmp.Result.Solana_core) == 0 {
82 | fmt.Println("Error in response for GetVersion: can't find solana core")
83 | return 0, 0, "", R_ERROR
84 | }
85 |
86 | tmp_chunks := strings.Split(tmp.Result.Solana_core, ".")
87 | version_major, _ := strconv.Atoi(tmp_chunks[0])
88 | version_minor, _ := strconv.Atoi(tmp_chunks[1])
89 |
90 | _ts := time.Now().UnixMilli()
91 | this.mu.Lock()
92 | this.version = tmp.Result.Solana_core
93 | this.version_ts = _ts
94 | this.mu.Unlock()
95 | return version_major, version_minor, tmp.Result.Solana_core, R_OK
96 | }
97 |
98 | func (this *SOLClient) GetBlock(block int, maxSupportedTransactionVersion int) ([]byte, ResponseType) {
99 | params := ""
100 | if maxSupportedTransactionVersion < 0 {
101 | params = fmt.Sprintf("[%d]", block)
102 | } else {
103 | params = fmt.Sprintf(`[%d,{"maxSupportedTransactionVersion":%d}]`, block, maxSupportedTransactionVersion)
104 | }
105 |
106 | ret := []byte("")
107 | r_type := R_OK
108 | if this.version_major == 1 && this.version_minor <= 6 {
109 | ret, r_type = this.RequestBasic("getConfirmedBlock", params)
110 | } else {
111 | ret, r_type = this.RequestBasic("getBlock", params)
112 | }
113 | if ret == nil {
114 | return ret, r_type
115 | }
116 |
117 | v := make(map[string]interface{})
118 | dec := json.NewDecoder(bytes.NewReader(ret))
119 | dec.UseNumber()
120 | dec.Decode(&v)
121 |
122 | switch v["result"].(type) {
123 | case nil:
124 | fmt.Sprintf("Warning: Cannot get block %d using endpoint %s. Probably the node doesn't have data.\n", block, this.endpoint)
125 | return ret, R_ERROR
126 | }
127 | return ret, R_OK
128 | }
129 |
130 | func (this *SOLClient) GetTransaction(hash string, maxSupportedTransactionVersion int) ([]byte, ResponseType) {
131 | params := ""
132 | if maxSupportedTransactionVersion < 0 {
133 | params = fmt.Sprintf(`["%s"]`, hash)
134 | } else {
135 | params = fmt.Sprintf(`["%s",{"maxSupportedTransactionVersion":%d}]`, hash, maxSupportedTransactionVersion)
136 | }
137 |
138 | ret := []byte("")
139 | r_type := ResponseType(R_OK)
140 | if this.version_major == 1 && this.version_minor <= 6 {
141 | ret, r_type = this.RequestBasic("getConfirmedTransaction", params)
142 | } else {
143 | ret, r_type = this.RequestBasic("getTransaction", params)
144 | }
145 | if ret == nil {
146 | return ret, r_type
147 | }
148 |
149 | // genesys patch, redo transaction if it fails for given transaction id
150 | if len(ret) < 200 &&
151 | bytes.Index(ret, ([]byte)("\"result\":null")) > -1 &&
152 | strings.Index(this.endpoint, "genesysgo") > -1 {
153 | ret, r_type = this.RequestBasic("getTransaction", params)
154 | if ret == nil {
155 | return ret, r_type
156 | }
157 | }
158 |
159 | v := make(map[string]interface{})
160 | dec := json.NewDecoder(bytes.NewReader(ret))
161 | dec.UseNumber()
162 | dec.Decode(&v)
163 |
164 | switch v["result"].(type) {
165 | case nil:
166 | return ret, R_ERROR
167 | }
168 | return ret, R_OK
169 | }
170 |
171 | func (this *SOLClient) SimpleCall(method, pubkey string, commitment string) ([]byte, ResponseType) {
172 | params := ""
173 | if len(commitment) > 0 {
174 | params = fmt.Sprintf("[\"%s\",\"%s\"]", pubkey, commitment)
175 | } else {
176 | params = fmt.Sprintf("[\"%s\"]", pubkey)
177 | }
178 |
179 | return this.RequestBasic(method, params)
180 | }
181 |
182 | func (this *SOLClient) GetBalance(pubkey string, commitment string) ([]byte, ResponseType) {
183 | return this.SimpleCall("getBalance", pubkey, commitment)
184 | }
185 |
186 | func (this *SOLClient) GetTokenSupply(pubkey string, commitment string) ([]byte, ResponseType) {
187 | return this.SimpleCall("getTokenSupply", pubkey, commitment)
188 | }
189 |
--------------------------------------------------------------------------------
/gosol/solana_proxy/client/stats.go:
--------------------------------------------------------------------------------
1 | package client
2 |
3 | import (
4 | "fmt"
5 | )
6 |
7 | type stat struct {
8 | stat_error_req int
9 | stat_error_resp int
10 | stat_error_resp_read int
11 | stat_error_json_decode int
12 | stat_error_json_marshal int
13 | stat_done int
14 | stat_ns_total uint64
15 |
16 | stat_request_by_fn map[string]int
17 | stat_bytes_received int
18 | stat_bytes_sent int
19 | }
20 |
21 | func (this *SOLClient) _statsGetAggr(seconds int) stat {
22 |
23 | s := stat{}
24 | _pos := this.stat_last_60_pos
25 | for i := 0; i < seconds; i++ {
26 | _pos--
27 | if _pos < 0 {
28 | _pos = 59
29 | }
30 |
31 | _tmp := this.stat_last_60[_pos%60]
32 | s.stat_done += _tmp.stat_done
33 | s.stat_error_json_decode += _tmp.stat_error_json_decode
34 | s.stat_error_json_marshal += _tmp.stat_error_json_marshal
35 | s.stat_error_req += _tmp.stat_error_req
36 | s.stat_error_resp += _tmp.stat_error_resp
37 | s.stat_error_resp_read += _tmp.stat_error_resp_read
38 | s.stat_ns_total += _tmp.stat_ns_total
39 |
40 | _tmp2 := make(map[string]int)
41 | for k, v := range _tmp.stat_request_by_fn {
42 | _tmp2[k] = _tmp2[k] + v
43 | }
44 | s.stat_request_by_fn = _tmp2
45 | s.stat_bytes_received += _tmp.stat_bytes_received
46 | s.stat_bytes_sent += _tmp.stat_bytes_sent
47 | }
48 |
49 | return s
50 | }
51 |
52 | func (this *SOLClient) _statsIsDead() (bool, int, int, string) {
53 |
54 | probe_isalive_seconds := this._probe_time * 2
55 | if probe_isalive_seconds < 30 {
56 | probe_isalive_seconds = 30
57 | }
58 | if probe_isalive_seconds > 60 {
59 | probe_isalive_seconds = 60
60 | }
61 |
62 | stat_requests := 0
63 | stat_errors := 0
64 | _pos := this.stat_last_60_pos
65 | for i := 0; i < probe_isalive_seconds; i++ {
66 | stat_requests += this.stat_last_60[_pos].stat_done
67 | stat_errors += this.stat_last_60[_pos].stat_error_resp
68 | stat_errors += this.stat_last_60[_pos].stat_error_resp_read
69 | stat_errors += this.stat_last_60[_pos].stat_error_json_decode
70 |
71 | _pos-- // take current second into account
72 | if _pos < 0 {
73 | _pos = 59
74 | }
75 | }
76 |
77 | // if we have no requests we assume something is wrong and we mark the node as dead
78 | // only if we're probing the node
79 | dead := this._probe_time == 0 && stat_errors*5 > stat_requests
80 | dead = dead || this._probe_time > 0 && stat_errors*5 >= stat_requests
81 |
82 | log := fmt.Sprintf("Health probing time %ds every %ds, Requests: %d, Errors: %d", probe_isalive_seconds, this._probe_time,
83 | stat_requests, stat_errors)
84 | return dead, stat_requests, stat_errors, log
85 | }
86 |
--------------------------------------------------------------------------------
/gosol/solana_proxy/client/status.go:
--------------------------------------------------------------------------------
1 | package client
2 |
3 | import (
4 | "fmt"
5 | node_status "gosol/solana_proxy/client/status"
6 | "gosol/solana_proxy/client/throttle"
7 | "html"
8 | "strings"
9 | "time"
10 |
11 | "github.com/slawomir-pryczek/HSServer/handler_socket2/hscommon"
12 | )
13 |
14 | var start_time = int64(0)
15 |
16 | func init() {
17 | start_time = time.Now().Unix()
18 | }
19 |
20 | func (this *SOLClient) GetStatus() string {
21 |
22 | status_throttle := throttle.ThrottleGoup(this.throttle).GetThrottleScore()
23 | out, status_description := node_status.Create(this.is_paused, status_throttle.Throttled, this.is_disabled)
24 |
25 | // Node name and status description
26 | {
27 | header := ""
28 | _t := "Private"
29 | if this.is_public_node {
30 | _t = "Public"
31 | }
32 | _e := this.endpoint
33 | _util := fmt.Sprintf("%.02f%%", float64(status_throttle.CapacityUsed)/100.0)
34 | header += fmt.Sprintf("%s Node #%d , Score: %d, Utilization: %s, %s\n", _t, this.id, status_throttle.Score, _util, _e)
35 | header += status_description
36 | header += "\n"
37 | header += this._probe_log
38 | out.SetHeader(header)
39 | }
40 |
41 | // Add basic badges
42 | if len(this.version) > 0 {
43 | out.AddBadge("Version: "+this.version, node_status.Gray, "Version number was updated on: "+time.UnixMilli(this.version_ts).Format("2006-01-02 15:04:05"))
44 | }
45 | if this.header != nil && len(this.header) > 0 {
46 | h_ := ""
47 | for k, v := range this.header {
48 | vv := strings.Join(v, ", ")
49 |
50 | out := vv
51 | if strings.Index(strings.ToLower(k), "authorization") != -1 && len(vv) > 5 {
52 | out = ""
53 | if strings.Index(strings.ToLower(vv), "bearer") == 0 {
54 | out += vv[0:6] + " "
55 | }
56 | out += "****" + vv[len(vv)-5:]
57 | }
58 | h_ += k + ": " + out + "", this.color))
76 | out = append(out, fmt.Sprintf("
%s
", this.icon_color, this.icon))
77 | out = append(out, "
"+this.header+" ")
78 |
79 | out = append(out, "
")
80 | for k, badge := range this.badge {
81 | info := this.badge_info[k]
82 | color := this.badge_color[k]
83 | out = append(out, fmt.Sprintf("
%s%s
", color, badge, info))
84 | }
85 | out = append(out, "
")
86 | out = append(out, "
")
89 | out = append(out, this.content)
90 | out = append(out, " ")
91 |
92 | return strings.Join(out, "")
93 | }
94 |
--------------------------------------------------------------------------------
/gosol/solana_proxy/client/throttle/group.go:
--------------------------------------------------------------------------------
1 | package throttle
2 |
3 | import (
4 | "math"
5 | )
6 |
7 | type ThrottleGoup []*Throttle
8 |
9 | func (this ThrottleGoup) OnRequest(function_name string) bool {
10 |
11 | for _, throttle := range this {
12 | if throttle.status_throttled {
13 | return false
14 | }
15 | }
16 |
17 | for _, throttle := range this {
18 | if !throttle.OnRequest(function_name) {
19 | return false
20 | }
21 | }
22 | return true
23 | }
24 |
25 | func (this ThrottleGoup) OnReceive(data_bytes int) {
26 | for _, throttle := range this {
27 | throttle.OnReceive(data_bytes)
28 | }
29 | }
30 |
31 | func (this ThrottleGoup) OnMaintenance(data_bytes int) {
32 | for _, throttle := range this {
33 | throttle.OnMaintenance(data_bytes)
34 | }
35 | }
36 |
37 | func (this ThrottleGoup) GetThrottleScore() ThrottleScore {
38 | ret := ThrottleScore{}
39 | ret.Score = math.MinInt64
40 | ret.Throttled = false
41 | ret.CapacityUsed = 0
42 | for _, throttle := range this {
43 |
44 | tmp := throttle.GetThrottleScore()
45 | if tmp.Score > ret.Score {
46 | ret.Score = tmp.Score
47 | }
48 | ret.Throttled = ret.Throttled || tmp.Throttled
49 | if tmp.CapacityUsed > ret.CapacityUsed {
50 | ret.CapacityUsed = tmp.CapacityUsed
51 | }
52 | }
53 |
54 | return ret
55 | }
56 |
57 | func (this ThrottleGoup) GetLimitsLeft() (int, int, int, int) {
58 |
59 | a, b, c, d := math.MaxInt64, math.MaxInt64, math.MaxInt64, 0
60 | for _, throttle := range this {
61 | a2, b2, c2, d2 := throttle.GetLimitsLeft()
62 |
63 | if a2 < a {
64 | a = a2
65 | }
66 | if b2 < b {
67 | b = b2
68 | }
69 | if c2 < c {
70 | c = c2
71 | }
72 | if d2 > d {
73 | d = d2
74 | }
75 | }
76 | return a, b, c, d
77 | }
78 |
79 | func (this ThrottleGoup) SetScoreModifier(m int) {
80 | for _, throttle := range this {
81 | throttle.score_modifier = m
82 | }
83 | }
84 |
--------------------------------------------------------------------------------
/gosol/solana_proxy/client/throttle/read_config.go:
--------------------------------------------------------------------------------
1 | package throttle
2 |
3 | import (
4 | "fmt"
5 | "strconv"
6 | "strings"
7 | )
8 |
9 | func MakeFromConfig(config string) ([]*Throttle, []string) {
10 | logs := make([]string, 0)
11 | error := func(a ...interface{}) ([]*Throttle, []string) {
12 | logs = append(logs, fmt.Sprint(a...))
13 | return nil, logs
14 | }
15 | log := func(a ...interface{}) {
16 | logs = append(logs, fmt.Sprint(a...))
17 | }
18 |
19 | type tw struct {
20 | time_window_size int
21 | time_window_count int
22 | }
23 | _get_timewindow_len := func(stat_time int) tw {
24 | if stat_time <= 120 {
25 | return tw{1, 120}
26 | }
27 | if stat_time <= 1200 {
28 | if stat_time%10 == 0 {
29 | return tw{10, 120}
30 | }
31 |
32 | log("Warning: throttling time >120s and not divisible by 10, may be suboptimal. Consider changing it")
33 | }
34 |
35 | start := stat_time / 100
36 | end := stat_time / 200
37 | for i := start; i < end; i++ {
38 | if stat_time%i == 0 {
39 | return tw{i, stat_time / i}
40 | }
41 | }
42 | log("Warning: throttling time can't be sliced into windows, consider using ",
43 | ((stat_time/120)+1)*120, " instead of ", stat_time)
44 | return tw{stat_time / 120, 125}
45 | }
46 | tws := make(map[tw]*Throttle, 0)
47 | _get_thr := func(throttle_time int) *Throttle {
48 | _k := _get_timewindow_len(throttle_time)
49 | if ret, ok := tws[_k]; ok {
50 | return ret
51 | }
52 | tws[_k] = MakeCustom(_k.time_window_count, _k.time_window_size)
53 | return tws[_k]
54 | }
55 |
56 | for _, v := range strings.Split(config, ";") {
57 | log("Processing throttle config: ", v)
58 | v := strings.Split(v, ",")
59 | if len(v) < 3 {
60 | return error("Error configuring throttling:", v, "... needs to have 3 parameters: type,limit,time_seconds")
61 | }
62 | for kk, vv := range v {
63 | v[kk] = strings.Trim(vv, "\r\n\t ")
64 | }
65 |
66 | if v[0] != "r" && v[0] != "f" && v[0] != "d" {
67 | return error("Error configuring throttling:", v, "... type needs to be [r]equests, [f]unctions, [d]ata received")
68 | }
69 |
70 | t_limit, _ := strconv.Atoi(v[1])
71 | t_time, _ := strconv.Atoi(v[2])
72 | thr := _get_thr(t_time)
73 | if t_limit <= 0 {
74 | return error("Error configuring throttling:", t_limit, "... limit needs to be > 0")
75 | }
76 | if t_time <= 0 {
77 | return error("Error configuring throttling:", t_time, "... time needs to be > 0")
78 | }
79 |
80 | if v[0] == "r" {
81 | thr.AddLimiter(L_REQUESTS, t_limit, t_time)
82 | log("Throttling requests ", t_limit, "/", t_time, "seconds")
83 | }
84 | if v[0] == "f" {
85 | thr.AddLimiter(L_REQUESTS_PER_FN, t_limit, t_time)
86 | log("Throttling requests for single function ", t_limit, "/", t_time, "seconds")
87 | }
88 | if v[0] == "d" {
89 | thr.AddLimiter(L_DATA_RECEIVED, t_limit, t_time)
90 | log("Throttling data received ", t_limit, "bytes/", t_time, "seconds")
91 | }
92 | }
93 |
94 | ret := make([]*Throttle, 0, len(tws))
95 | for _, v := range tws {
96 | ret = append(ret, v)
97 | }
98 | return ret, logs
99 | }
100 |
101 | func MakeForPublic() ([]*Throttle, []string) {
102 | thr := Make()
103 | thr.AddLimiter(L_REQUESTS, 70, 10)
104 | thr.AddLimiter(L_REQUESTS_PER_FN, 20, 10)
105 | thr.AddLimiter(L_DATA_RECEIVED, 75*1000*1000, 30)
106 |
107 | return []*Throttle{thr}, []string{"Adding standard throttle for public nodes"}
108 | }
109 |
--------------------------------------------------------------------------------
/gosol/solana_proxy/client/throttle/stats.go:
--------------------------------------------------------------------------------
1 | package throttle
2 |
3 | type stat struct {
4 | stat_requests int
5 | stat_request_by_fn map[string]int
6 | stat_data_received int
7 | }
8 |
9 | func (this *Throttle) OnRequest(function_name string) bool {
10 | if this.status_throttled {
11 | return false
12 | }
13 |
14 | to_mod := &this.stats[this.stats_pos]
15 | to_mod.stat_requests++
16 | to_mod.stat_request_by_fn[function_name]++
17 |
18 | // update statistics
19 | tmp := this._getThrottleScore()
20 | this.status_throttled = tmp.Throttled
21 | this.status_score = tmp.Score
22 | this.status_capacity_used = tmp.CapacityUsed
23 | return true
24 | }
25 |
26 | func (this *Throttle) OnReceive(data_bytes int) {
27 | to_mod := &this.stats[this.stats_pos]
28 | to_mod.stat_data_received += data_bytes
29 | }
30 |
31 | func (this *Throttle) OnMaintenance(ts int) {
32 | new_pos := (ts / this.stats_window_size_seconds) % len(this.stats)
33 | if new_pos == this.stats_pos {
34 | return
35 | }
36 |
37 | this.stats_pos = new_pos
38 | this.stats[this.stats_pos].stat_requests = 0
39 | this.stats[this.stats_pos].stat_request_by_fn = make(map[string]int)
40 | this.stats[this.stats_pos].stat_data_received = 0
41 |
42 | // update statistics, as data is changing
43 | tmp := this._getThrottleScore()
44 | this.status_throttled = tmp.Throttled
45 | this.status_score = tmp.Score
46 | this.status_capacity_used = tmp.CapacityUsed
47 | }
48 |
49 | // Get throttle status, first return parameter is amount, second amount used 0-10000
50 | func (this *Throttle) _getThrottleStatus(l *Limiter) (int, int) {
51 |
52 | amt := 0
53 | pos := this.stats_pos
54 |
55 | if l.t == L_REQUESTS {
56 | for i := 0; i < l.in_time_windows; i++ {
57 | amt += this.stats[pos].stat_requests
58 | pos--
59 | if pos < 0 {
60 | pos = len(this.stats) - 1
61 | }
62 | }
63 | }
64 |
65 | if l.t == L_DATA_RECEIVED {
66 | for i := 0; i < l.in_time_windows; i++ {
67 | amt += this.stats[pos].stat_data_received
68 | pos--
69 | if pos < 0 {
70 | pos = len(this.stats) - 1
71 | }
72 | }
73 | }
74 |
75 | if l.t == L_REQUESTS_PER_FN {
76 | tmp := make(map[string]int)
77 |
78 | for i := 0; i < l.in_time_windows; i++ {
79 | for k, v := range this.stats[pos].stat_request_by_fn {
80 | tmp[k] += v
81 | }
82 | pos--
83 | if pos < 0 {
84 | pos = len(this.stats) - 1
85 | }
86 | }
87 |
88 | for _, v := range tmp {
89 | if amt < v {
90 | amt = v
91 | }
92 | }
93 | }
94 |
95 | percentage_used := 0
96 | if amt >= l.maximum {
97 | percentage_used = 10000
98 | } else {
99 | percentage_used = int((float64(amt) * 10000) / float64(l.maximum))
100 | }
101 |
102 | return amt, percentage_used
103 | }
104 |
105 | // Get Score 0-10000
106 | type ThrottleScore struct {
107 | Score int
108 | Throttled bool
109 | CapacityUsed int
110 | }
111 |
112 | func (this *Throttle) _getThrottleScore() ThrottleScore {
113 |
114 | // for non-throttled nodes get score based on last 10 seconds of data
115 | // every request is worth 1 point
116 | // every 10kb of data is worth 1 point
117 | if len(this.limiters) == 0 {
118 | pos := this.stats_pos
119 | score := 0
120 | for i := 0; i < 10; i++ {
121 | score += this.stats[pos].stat_requests + (this.stats[pos].stat_data_received / 10000)
122 | pos--
123 | if pos < 0 {
124 | pos = len(this.stats) - 1
125 | }
126 | }
127 | score += this.score_modifier
128 | return ThrottleScore{score, false, 0}
129 | }
130 |
131 | score := 0
132 | disabled := false
133 | for k, _ := range this.limiters {
134 | _, tmp := this._getThrottleStatus(&this.limiters[k])
135 | if tmp > score {
136 | score = tmp
137 | }
138 | }
139 | capacity_used := score
140 | if score >= 10000 {
141 | disabled = true
142 | }
143 |
144 | score += this.score_modifier
145 | return ThrottleScore{score, disabled, capacity_used}
146 | }
147 |
148 | func (this *Throttle) GetThrottleScore() ThrottleScore {
149 | return ThrottleScore{this.status_score, this.status_throttled, this.status_capacity_used}
150 | }
151 |
--------------------------------------------------------------------------------
/gosol/solana_proxy/client/throttle/status.go:
--------------------------------------------------------------------------------
1 | package throttle
2 |
3 | import (
4 | "fmt"
5 | "github.com/slawomir-pryczek/HSServer/handler_socket2/hscommon"
6 | "strings"
7 | )
8 |
9 | /* This has to hold mutex externally */
10 | func (this *Throttle) GetStatus() string {
11 |
12 | _progress := func(p int) string {
13 | if p > 100 {
14 | p = 100
15 | }
16 | p = p / 10
17 |
18 | ret := strings.Repeat("◆", p)
19 | if p < 10 && 10-p > 0 {
20 | ret = ret + strings.Repeat("◇", 10-p)
21 | }
22 | return ret
23 | }
24 |
25 | status := " ⬤ Throttling disabled (##layout##) ⏵︎⏵︎⏵︎ "
26 | if (len(this.limiters) > 0) && this.status_throttled {
27 | status = " ⮿ Throttling group (##layout##), exhausted "
28 | }
29 | if (len(this.limiters) > 0) && !this.status_throttled {
30 | status = "⬤ Throttling group (##layout##)%s %d/%d (%.01f%%) +%d " + ret + " "
80 | })
81 | }
82 |
83 | func _run_custom_check() {
84 |
85 | max_age_ms := cc.max_data_age_ms
86 | max_block_lag := cc.max_block_lag
87 |
88 | mu.RLock()
89 | infos := make([]*client.Solclientinfo, 0, len(clients))
90 | for _, client := range clients {
91 | infos = append(infos, client.GetInfo())
92 | }
93 | mu.RUnlock()
94 |
95 | status := make(map[int]string, 0)
96 | max_block := 0
97 | wg := sync.WaitGroup{}
98 | info_mutex := sync.Mutex{}
99 | for num, info := range infos {
100 | _age_ms := time.Now().UnixMilli() - info.Available_block_last_ts
101 | if _age_ms < max_age_ms {
102 | continue
103 | }
104 |
105 | // Check if we still can run this client
106 | _client := (*client.SOLClient)(nil)
107 | mu.RLock()
108 | if num < len(clients) {
109 | wg.Add(1)
110 | _client = clients[num]
111 |
112 | _comm := "Never updated"
113 | if _age_ms < 24*3600*1000 {
114 | _comm = fmt.Sprintf("%.2fs", float64(_age_ms)/1000.0)
115 | }
116 | status[num] = fmt.Sprintf("(Trying refresh - %s)", _comm)
117 | }
118 | mu.RUnlock()
119 | if _client == nil {
120 | continue
121 | }
122 |
123 | // Get max blocks for all clients which need it
124 | num := num
125 | go func() {
126 | _client.GetLastAvailableBlock()
127 | _info := _client.GetInfo()
128 | info_mutex.Lock()
129 | infos[num] = _info
130 | info_mutex.Unlock()
131 | wg.Done()
132 | }()
133 | }
134 | wg.Wait()
135 |
136 | //Get maximum block
137 | for _, info := range infos {
138 | if info.Available_block_last > max_block {
139 | max_block = info.Available_block_last
140 | }
141 | }
142 |
143 | mu.RLock()
144 | defer mu.RUnlock()
145 |
146 | log := ""
147 | for num, info := range infos {
148 | is_ok := max_block-info.Available_block_last <= max_block_lag
149 | _is_ok := "OK "
150 | if !is_ok {
151 | _is_ok = "LAGGING"
152 | }
153 |
154 | for _, client := range clients {
155 | if strings.Compare(client.GetEndpoint(), info.Endpoint) == 0 {
156 | if is_ok {
157 | client.SetPaused(!is_ok, "")
158 | continue
159 | }
160 |
161 | if info.Available_block_last_ts == 0 {
162 | client.SetPaused(!is_ok, "Paused by Custom Health Checker.\nCan't get last block")
163 | continue
164 | }
165 | client.SetPaused(!is_ok, fmt.Sprintf("Paused by Custom Health Checker.\nNode is lagging behind %d blocks (%d max)",
166 | max_block-info.Available_block_last, max_block_lag))
167 | }
168 | }
169 |
170 | _age_ms := float64((time.Now().UnixMilli() - info.Available_block_last_ts)) / 1000
171 | _diff := max_block - info.Available_block_last
172 |
173 | log += fmt.Sprintf("Node #%d %s Score: %d, Highest Block: %d/%d Max (%d diff) (%.2fs Age) %s\n",
174 | num, _is_ok, info.Score,
175 | info.Available_block_last, max_block, _diff,
176 | _age_ms, status[num])
177 | }
178 |
179 | cc.mu.Lock()
180 | cc._log = log
181 | cc.mu.Unlock()
182 | }
183 |
--------------------------------------------------------------------------------
/gosol/solana_proxy/manager.go:
--------------------------------------------------------------------------------
1 | package solana_proxy
2 |
3 | import (
4 | "gosol/solana_proxy/client"
5 | "math"
6 | "sync"
7 | )
8 |
9 | var mu sync.RWMutex
10 | var clients []*client.SOLClient
11 |
12 | func init() {
13 | clients = make([]*client.SOLClient, 0, 10)
14 | }
15 |
16 | func ClientRegister(c *client.SOLClient) {
17 | ClientManage(c, math.MaxUint64)
18 | }
19 |
20 | func ClientRemove(id uint64) bool {
21 | return ClientManage(nil, id)
22 | }
23 |
24 | func ClientManage(add *client.SOLClient, removeClientID uint64) bool {
25 | acted := false
26 |
27 | mu.Lock()
28 | defer mu.Unlock()
29 |
30 | if add != nil && removeClientID == math.MaxUint64 {
31 | clients = append(clients, add)
32 | return true
33 | }
34 |
35 | tmp := make([]*client.SOLClient, 0, len(clients))
36 | for _, client := range clients {
37 | if client.GetInfo().ID == removeClientID {
38 | acted = true
39 | if add != nil {
40 | tmp = append(tmp, add)
41 | add = nil
42 | }
43 | continue
44 | }
45 | tmp = append(tmp, client)
46 | }
47 | if add != nil {
48 | tmp = append(tmp, add)
49 | }
50 |
51 | clients = tmp
52 | return acted
53 | }
54 |
55 | func GetMinMaxBlocks() (int, int, int, int) {
56 |
57 | mu.RLock()
58 | defer mu.RUnlock()
59 |
60 | // a public; b private
61 | a, b, c, d := -1, -1, -1, -1
62 | for _, v := range clients {
63 | info := v.GetInfo()
64 | if info.Is_disabled {
65 | continue
66 | }
67 |
68 | if info.Is_public_node {
69 | if a == -1 || info.Available_block_first > a {
70 | a = info.Available_block_first
71 | }
72 | if c == -1 || info.Available_block_last < c {
73 | c = info.Available_block_last
74 | }
75 | } else {
76 | if b == -1 || info.Available_block_first > b {
77 | b = info.Available_block_first
78 | }
79 | if d == -1 || info.Available_block_last < d {
80 | d = info.Available_block_last
81 | }
82 | }
83 | }
84 | return a, b, c, d
85 | }
86 |
--------------------------------------------------------------------------------
/gosol/solana_proxy/scheduler.go:
--------------------------------------------------------------------------------
1 | package solana_proxy
2 |
3 | import (
4 | "gosol/solana_proxy/client"
5 | "sort"
6 | )
7 |
8 | type scheduler struct {
9 | min_block_no int
10 | clients []*client.SOLClient
11 | force_public bool
12 | force_private bool
13 | }
14 |
15 | func MakeScheduler() *scheduler {
16 |
17 | mu.RLock()
18 | tmp := make([]*client.SOLClient, len(clients))
19 | copy(tmp, clients)
20 | mu.RUnlock()
21 |
22 | ret := &scheduler{min_block_no: -1, clients: tmp}
23 | ret.force_public = false
24 | ret.force_private = false
25 | return ret
26 | }
27 |
28 | func (this *scheduler) SetMinBlock(min_block_no int) {
29 | this.min_block_no = min_block_no
30 | }
31 |
32 | func (this *scheduler) ForcePublic(f bool) {
33 | this.force_public = f
34 | if this.force_public && this.force_private {
35 | this.force_public = false
36 | this.force_private = false
37 | }
38 | }
39 | func (this *scheduler) ForcePrivate(f bool) {
40 | this.force_private = f
41 | if this.force_public && this.force_private {
42 | this.force_public = false
43 | this.force_private = false
44 | }
45 | }
46 |
47 | /* Gets client, prioritize private client */
48 | func (this *scheduler) GetAnyClient() *client.SOLClient {
49 | return this._pick_next()
50 | }
51 |
52 | /* Get public client only */
53 | func (this *scheduler) GetPublicClient() *client.SOLClient {
54 |
55 | // we forced something, so override the client returned
56 | if this.force_public || this.force_private {
57 | return this._pick_next()
58 | }
59 |
60 | this.force_public = true
61 | ret := this._pick_next()
62 | this.force_public = false
63 | return ret
64 | }
65 |
66 | func (this *scheduler) GetAll(is_public bool, include_deactivated bool) []*client.SOLClient {
67 |
68 | ret := make([]*client.SOLClient, 0, len(this.clients))
69 | for _, v := range this.clients {
70 | info := v.GetInfo()
71 | if (info.Is_disabled || info.Is_throttled || info.Is_paused) && include_deactivated == false {
72 | continue
73 | }
74 |
75 | if is_public != info.Is_public_node {
76 | continue
77 | }
78 | if this.min_block_no > -1 && this.min_block_no <= info.Available_block_first {
79 | continue
80 | }
81 | ret = append(ret, v)
82 | }
83 | return ret
84 | }
85 |
86 | func (this *scheduler) GetAllSorted(is_public bool, include_disabled bool) []*client.SOLClient {
87 |
88 | ret := this.GetAll(is_public, include_disabled)
89 | type r_sort struct {
90 | c *client.SOLClient
91 | score int
92 | }
93 | s := make([]r_sort, 0, len(ret))
94 | for _, v := range ret {
95 | s = append(s, r_sort{v, v.GetInfo().Score})
96 | }
97 | sort.Slice(s, func(a, b int) bool {
98 | return s[a].score < s[b].score
99 | })
100 | for k, v := range s {
101 | ret[k] = v.c
102 | }
103 |
104 | return ret
105 | }
106 |
--------------------------------------------------------------------------------
/gosol/solana_proxy/scheduler_pick.go:
--------------------------------------------------------------------------------
1 | package solana_proxy
2 |
3 | import (
4 | "gosol/solana_proxy/client"
5 | )
6 |
7 | func (this *scheduler) _pick_next() *client.SOLClient {
8 |
9 | min, min_pos := -1, -1
10 | for num, v := range this.clients {
11 | if v == nil {
12 | continue
13 | }
14 |
15 | info := v.GetInfo()
16 | if info.Is_disabled || info.Is_throttled || info.Is_paused {
17 | this.clients[num] = nil
18 | continue
19 | }
20 |
21 | if info.Is_public_node && this.force_private {
22 | continue
23 | }
24 | if !info.Is_public_node && this.force_public {
25 | continue
26 | }
27 |
28 | if this.min_block_no > -1 && this.min_block_no <= info.Available_block_first {
29 | continue
30 | }
31 |
32 | _r := info.Score
33 | if min == -1 || _r < min {
34 | min = _r
35 | min_pos = num
36 | }
37 | }
38 |
39 | if min_pos == -1 {
40 | return nil
41 | }
42 | ret := this.clients[min_pos]
43 | this.clients[min_pos] = nil
44 | return ret
45 | }
46 |
--------------------------------------------------------------------------------
/gosol/solana_proxy/status.go:
--------------------------------------------------------------------------------
1 | package solana_proxy
2 |
3 | import (
4 | "github.com/slawomir-pryczek/HSServer/handler_socket2"
5 | )
6 |
7 | func init() {
8 |
9 | get_status := func() (string, string) {
10 |
11 | info := "This section represents individual SOLANA nodes, with number of requests and errors\n"
12 | info += "Err JM - Json Marshall error. We were unable to build JSON payload required for your request\n"
13 | info += "Err Req - Request Error. We were unable to send request to host\n"
14 | info += "Err Resp - Response Error. We were unable to get server response\n"
15 | info += "Err RResp - Response Reading Error. We were unable to read server response\n"
16 | info += "Err Decode - Json Decode Error. We were unable read received JSON\n"
17 |
18 | status := ""
19 | sh := MakeScheduler()
20 | for _, v := range sh.GetAll(true, true) {
21 | status += v.GetStatus()
22 | }
23 | for _, v := range sh.GetAll(false, true) {
24 | status += v.GetStatus()
25 | }
26 |
27 | return "Solana Proxy", "" + info + status + " "
28 | }
29 |
30 | handler_socket2.StatusPluginRegister(get_status)
31 | }
32 |
--------------------------------------------------------------------------------
/gosol/solana_proxy/throttle/throttle.go:
--------------------------------------------------------------------------------
1 | package throttle
2 |
3 | import (
4 | "encoding/json"
5 | "fmt"
6 | )
7 |
8 | type Throttle struct {
9 | throttling_enabled bool
10 |
11 | requests int
12 | requests_per_fn_max int
13 | data_received int
14 | }
15 |
16 | type _throttleConfig struct {
17 | Throttle_requests int
18 | Throttle_requests_per_fn_max int
19 | Throttle_data_received int
20 |
21 | Throttle_s_requests int
22 | Throttle_s_requests_per_fn_max int
23 | Throttle_s_data_received int
24 | }
25 |
26 | var ThrottleConfig = _throttleConfig{}
27 |
28 | func init() {
29 | ThrottleConfig.Throttle_requests = 90
30 | ThrottleConfig.Throttle_requests_per_fn_max = 33
31 | ThrottleConfig.Throttle_data_received = 1000 * 1000 * 95
32 |
33 | ThrottleConfig.Throttle_s_requests = 12
34 | ThrottleConfig.Throttle_s_requests_per_fn_max = 12
35 | ThrottleConfig.Throttle_s_data_received = 32
36 | }
37 |
38 | func Make(is_public_node bool, requests, requests_per_fn_max, data_received int) Throttle {
39 | return Throttle{is_public_node, requests, requests_per_fn_max, data_received}
40 | }
41 |
42 | func (this Throttle) GetUsedCapacity() float64 {
43 | if !this.throttling_enabled {
44 | return 0
45 | }
46 |
47 | tmp := float64(0)
48 | tmp2 := float64(0)
49 | tmp2 = float64(this.requests) * 100 / float64(ThrottleConfig.Throttle_requests)
50 | if tmp2 > tmp {
51 | tmp = tmp2
52 | }
53 | tmp2 = float64(this.requests_per_fn_max) * 100 / float64(ThrottleConfig.Throttle_requests_per_fn_max)
54 | if tmp2 > tmp {
55 | tmp = tmp2
56 | }
57 | tmp2 = float64(this.data_received) * 100 / float64(ThrottleConfig.Throttle_data_received)
58 | if tmp2 > tmp {
59 | tmp = tmp2
60 | }
61 |
62 | _tmp := int(tmp * 1000.0)
63 | return float64(_tmp/100) / 10.0
64 | }
65 |
66 | func (this Throttle) IsThrottled() ([]byte, string) {
67 | if !this.throttling_enabled {
68 | return nil, "Throttling disabled."
69 | }
70 |
71 | throttled_comment := ""
72 | if this.requests >= ThrottleConfig.Throttle_requests {
73 | throttled_comment = fmt.Sprintf("Too many requests %d/%d", this.requests, ThrottleConfig.Throttle_requests)
74 | }
75 | if this.requests_per_fn_max >= ThrottleConfig.Throttle_requests_per_fn_max {
76 | throttled_comment = fmt.Sprintf("Too many requests for single method %d/%d", this.requests_per_fn_max, ThrottleConfig.Throttle_requests_per_fn_max)
77 | }
78 | if this.data_received >= ThrottleConfig.Throttle_data_received {
79 | throttled_comment = fmt.Sprintf("Too much data received %d/%d", this.data_received, ThrottleConfig.Throttle_data_received)
80 | }
81 | if len(throttled_comment) == 0 {
82 | return nil, "Throttling enabled. This node is not throttled."
83 | }
84 |
85 | ret := make(map[string]interface{})
86 | ret["error"] = "Throttled public node, please wait"
87 | ret["throttled"] = true
88 | ret["throttled_comment"] = throttled_comment
89 | ret["throttle_info"] = nil
90 | ret["throttle_timespan_seconds"] = 12
91 |
92 | tmp := map[string]interface{}{}
93 | tmp2 := map[string]interface{}{}
94 | tmp2["value"] = this.requests
95 | tmp2["max"] = ThrottleConfig.Throttle_requests
96 | tmp2["description"] = "requests made"
97 | tmp["requests"] = tmp2
98 |
99 | tmp2 = map[string]interface{}{}
100 | tmp2["value"] = this.requests_per_fn_max
101 | tmp2["max"] = ThrottleConfig.Throttle_requests_per_fn_max
102 | tmp2["description"] = "requests made calling single function"
103 | tmp["requests_fn"] = tmp2
104 |
105 | tmp2 = map[string]interface{}{}
106 | tmp2["value"] = this.data_received
107 | tmp2["max"] = ThrottleConfig.Throttle_data_received
108 | tmp2["description"] = "bytes received"
109 | tmp["received"] = tmp2
110 | ret["throttle_info"] = tmp
111 |
112 | r, err := json.Marshal(ret)
113 | if err != nil {
114 | return []byte(`{"throttled":true"}`), throttled_comment
115 | }
116 | return r, throttled_comment
117 | }
118 |
119 | func (this Throttle) GetThrottledStatus() map[string]interface{} {
120 |
121 | ret := map[string]interface{}{}
122 |
123 | throttled_data, throttled_comment := this.IsThrottled()
124 | if !this.throttling_enabled {
125 | ret["throttled_comment"] = throttled_comment
126 | ret["is_throttled"] = false
127 | ret["p_capacity_used"] = float64(0)
128 | return ret
129 | }
130 |
131 | ret["throttled_comment"] = throttled_comment
132 | ret["is_throttled"] = throttled_data != nil
133 |
134 | ret["p_capacity_used"] = this.GetUsedCapacity()
135 | ret["throttle_0"] = fmt.Sprintf("Throttle requests (last %d seconds): %d/%d",
136 | ThrottleConfig.Throttle_s_requests, this.requests, ThrottleConfig.Throttle_requests)
137 | ret["throttle_1"] = fmt.Sprintf("Throttle requests for single method (last %d seconds): %d/%d",
138 | ThrottleConfig.Throttle_s_requests_per_fn_max, this.requests_per_fn_max, ThrottleConfig.Throttle_requests_per_fn_max)
139 | ret["throttle_2"] = fmt.Sprintf("Throttle data received (last %d seconds): %.02fMB / %.02fMB",
140 | ThrottleConfig.Throttle_s_data_received, float64(this.data_received)/1000/1000, float64(ThrottleConfig.Throttle_data_received)/1000/1000)
141 | return ret
142 | }
143 |
--------------------------------------------------------------------------------
/handler_socket2/byteslabs/byteslabs_test.go:
--------------------------------------------------------------------------------
1 | package byteslabs
2 |
3 | import (
4 | "bytes"
5 | "fmt"
6 | "math/rand"
7 | "runtime"
8 | "runtime/debug"
9 | "sync"
10 | "sync/atomic"
11 | "testing"
12 | "time"
13 | )
14 |
15 | var rnd_gen []int
16 | var rnd_pos int32
17 |
18 | func init() {
19 | rnd_gen = make([]int, 50000)
20 | for i := 0; i < len(rnd_gen); i++ {
21 | rnd_gen[i] = rand.Int()
22 | }
23 | }
24 |
25 | func rand_b(n int) int {
26 | pos := int(atomic.AddInt32(&rnd_pos, 1))
27 | return rnd_gen[pos%len(rnd_gen)] % n
28 | }
29 |
30 | func TestByteslabs(t *testing.T) {
31 |
32 | ts := time.Now().UnixMilli()
33 |
34 | var letterRunes = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
35 | var randomStr = []byte(nil)
36 | rnd_init := func(n int) []byte {
37 | b := make([]byte, n)
38 | for i := range b {
39 | b[i] = letterRunes[rand_b(len(letterRunes))]
40 | }
41 | return b
42 | }
43 | randomStr = rnd_init(100000)
44 | rnd := func(n int) []byte {
45 | start := rand.Intn(1000)
46 | return randomStr[start : start+n]
47 | }
48 |
49 | runtime.GOMAXPROCS(60)
50 |
51 | single_pass := func(disp bool) {
52 | alloc := MakeAllocator()
53 |
54 | els_ok, els_fail, els_len := 0, 0, 0
55 | stored := make([][]byte, 0, 0)
56 | n_strings := rand.Intn(200) + 10
57 |
58 | for i := 0; i < n_strings; i++ {
59 | n := 0
60 | if i%10 == 0 {
61 | n = rand.Intn(60000) + 30000
62 | } else {
63 | n = rand.Intn(600)
64 | }
65 |
66 | tostore := rnd(n)
67 | els_len += n
68 | stored = append(stored, tostore)
69 | if i%2 == 0 {
70 | // store test1
71 | //_sa := make([]byte, n)
72 | _sa := alloc.Allocate(n)[0:n]
73 | copy(_sa, tostore)
74 | stored = append(stored, _sa)
75 | } else {
76 | //_sa := make([]byte, 0, n)
77 | _sa := alloc.Allocate(n)
78 | _sa_b := _sa[0:n]
79 | for _, b := range tostore {
80 | _sa = append(_sa, b)
81 | }
82 | stored = append(stored, _sa_b)
83 | }
84 |
85 | for i := 0; i < len(stored); i += 2 {
86 | ok := bytes.Compare(stored[i], stored[i+1]) == 0
87 | if ok {
88 | els_ok++
89 | } else {
90 | els_fail++
91 | fmt.Println("!! ", len(stored[i]), len(stored[i+1]))
92 | }
93 | }
94 | }
95 | alloc.Release()
96 | if disp {
97 | fmt.Println(n_strings, "Strings... ", "OK: ", els_ok, " Fail", els_fail, " Bytes: ", els_len)
98 | }
99 |
100 | }
101 |
102 | threads := 10
103 | for z := 0; z < 60/threads; z++ {
104 | wg := sync.WaitGroup{}
105 | for i := 0; i < threads; i++ {
106 | i := i
107 | wg.Add(1)
108 | go func() {
109 | fmt.Println("Started : ", i)
110 | for i := 0; i < 300; i++ {
111 | single_pass(i%60 == 0)
112 | }
113 | wg.Done()
114 | }()
115 | }
116 | wg.Wait()
117 | }
118 |
119 | fmt.Println(float64(time.Now().UnixMilli()-ts) / 1000)
120 |
121 | var garC debug.GCStats
122 | debug.ReadGCStats(&garC)
123 | fmt.Printf("\nLastGC:\t%s", garC.LastGC) // time of last collection
124 | fmt.Printf("\nNumGC:\t%d", garC.NumGC) // number of garbage collections
125 | fmt.Printf("\nPauseTotal:\t%s", garC.PauseTotal) // total pause for all collections
126 |
127 | }
128 |
129 | func BenchmarkByteslabs(b *testing.B) {
130 | ts := time.Now().UnixMilli()
131 | runtime.GOMAXPROCS(60)
132 |
133 | single_pass := func(disp bool) {
134 | alloc := MakeAllocator()
135 | n_strings := rand_b(200) + 10
136 |
137 | for i := 0; i < n_strings; i++ {
138 | n := 0
139 | if i%10 == 0 {
140 | n = rand_b(60000) + 30000
141 | } else {
142 | n = rand_b(600)
143 | }
144 | alloc.Allocate(n)
145 | }
146 | alloc.Release()
147 | }
148 |
149 | threads := 60
150 | fmt.Println("Startig threads: ")
151 | for z := 0; z < 60/threads; z++ {
152 | wg := sync.WaitGroup{}
153 | for i := 0; i < threads; i++ {
154 | i := i
155 | wg.Add(1)
156 | go func() {
157 | fmt.Print(i, " ")
158 | for i := 0; i < 1000; i++ {
159 | single_pass(i%60 == 0)
160 | }
161 | wg.Done()
162 | }()
163 | }
164 | wg.Wait()
165 | }
166 | fmt.Println()
167 |
168 | fmt.Println(float64(time.Now().UnixMilli()-ts) / 1000)
169 |
170 | var garC debug.GCStats
171 | debug.ReadGCStats(&garC)
172 | fmt.Printf("\nLastGC:\t%s", garC.LastGC) // time of last collection
173 | fmt.Printf("\nNumGC:\t%d", garC.NumGC) // number of garbage collections
174 | fmt.Printf("\nPauseTotal:\t%s", garC.PauseTotal) // total pause for all collections
175 |
176 | b.ReportAllocs()
177 | fmt.Println(GetStatusStr())
178 | }
179 |
--------------------------------------------------------------------------------
/handler_socket2/byteslabs/status.go:
--------------------------------------------------------------------------------
1 | package byteslabs
2 |
3 | import (
4 | "fmt"
5 | "strconv"
6 | "strings"
7 |
8 | "github.com/slawomir-pryczek/HSServer/handler_socket2/hscommon"
9 | )
10 |
11 | func GetStatus() (string, string) {
12 | ret := ""
13 | ret += fmt.Sprintf("Slab Allocator. Slab Size: %d x %d Slabs = %d items per Page. %d Pages.\n", slab_size, slab_count, slab_size*slab_count, mem_chunks_count)
14 | ret += "Alloc Full - We're allocating >1 SLABS, happens at page begin\n"
15 | ret += "Alloc Full Small - We're allocating 1 SLAB, happens at page's end to prevent fragmentation\n"
16 | ret += "Tail - We can put the data into already allocated SLAB's tail\n"
17 | ret += "OOM - Page is out of memory, we're using system allocator instead\n"
18 | ret += "Routed - Allocations was routed to least used slab page, allocations per routing\n"
19 | ret += "Routed Alloc - Allocations in routed slabs\n"
20 | ret += ". - Slab is EMPTY; F - Slab is FULL\n"
21 | ret += " "
22 |
23 | ret += ""
29 |
30 | tg := hscommon.NewTableGen("#", "Alloc Full", "Full Small", "Tail",
31 | "OOM ", "Slabs", "Routed ", "Routed Alloc")
32 | tg.SetClass("tab salloc")
33 |
34 | for k, chunk := range mem_chunks {
35 | chunk.mu.Lock()
36 |
37 | _slabs := ""
38 | for _, used := range chunk.slab_used {
39 | if used {
40 | _slabs += "F"
41 | } else {
42 | _slabs += "."
43 | }
44 | }
45 |
46 | percent_oom := int(chunk.stat_alloc_full + chunk.stat_alloc_full_small + chunk.stat_alloc_tail)
47 | if percent_oom > 0 {
48 | percent_oom = int((float64(chunk.stat_oom) / float64(percent_oom)) * 1000)
49 | }
50 | oom := fmt.Sprintf("%d (%d.%d%%)", chunk.stat_oom, percent_oom/10, percent_oom%10)
51 |
52 | apr := 0
53 | if chunk.stat_routed > 0 {
54 | apr = chunk.stat_routed_alloc * 10 / chunk.stat_routed
55 | }
56 | routed := fmt.Sprintf("%d (%d.%d apr)", chunk.stat_routed, apr/10, apr%10)
57 |
58 | tg.AddRow(strconv.Itoa(k), strconv.Itoa(chunk.stat_alloc_full), strconv.Itoa(chunk.stat_alloc_full_small),
59 | strconv.Itoa(chunk.stat_alloc_tail), oom, ""+_slabs+" ", routed, strconv.Itoa(chunk.stat_routed_alloc))
60 |
61 | chunk.mu.Unlock()
62 | }
63 |
64 | return "Slab Allocator \\ QCompress", ret + tg.Render()
65 | }
66 |
67 | func GetStatusStr() string {
68 | ret := make([]string, 0, 40)
69 | ret = append(ret, "=====")
70 | total_failed := 0
71 | total_f, total_fs, total_t := 0, 0, 0
72 | for k, v := range mem_chunks {
73 | ret = append(ret, fmt.Sprint(k, "Full:", v.stat_alloc_full, "Full Small:", v.stat_alloc_full_small,
74 | "Tail:", v.stat_alloc_tail, "OOM:", v.stat_oom, "Routed:", v.stat_routed,
75 | "Slab taken:", v.used_slab_count))
76 |
77 | total_failed += v.stat_oom - v.stat_routed
78 | total_f += v.stat_alloc_full
79 | total_fs += v.stat_alloc_full_small
80 | total_t += v.stat_alloc_tail
81 | }
82 |
83 | ret = append(ret, fmt.Sprintf("=Totals ... Full: %d Full Small: %d Tail: %d \n", total_f, total_fs, total_t))
84 | ret = append(ret, fmt.Sprintf("=Items Total: %d, Failed: %d\n",
85 | total_f+total_fs+total_t+total_failed, total_failed))
86 | return strings.Join(ret, "\n")
87 | }
88 |
--------------------------------------------------------------------------------
/handler_socket2/byteslabs2/byteslabs_test.go:
--------------------------------------------------------------------------------
1 | package byteslabs2
2 |
3 | import (
4 | "bytes"
5 | "fmt"
6 | "math/rand"
7 | "runtime"
8 | "runtime/debug"
9 | "sync"
10 | "sync/atomic"
11 | "testing"
12 | "time"
13 | )
14 |
15 | var rnd_gen []int
16 | var rnd_pos int32
17 |
18 | func init() {
19 | rnd_gen = make([]int, 50000)
20 | for i := 0; i < len(rnd_gen); i++ {
21 | rnd_gen[i] = rand.Int()
22 | }
23 | }
24 |
25 | func rand_b(n int) int {
26 | pos := int(atomic.AddInt32(&rnd_pos, 1))
27 | return rnd_gen[pos%len(rnd_gen)] % n
28 | }
29 |
30 | func TestByteslabs(t *testing.T) {
31 |
32 | ts := time.Now().UnixMilli()
33 |
34 | var letterRunes = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
35 | var randomStr = []byte(nil)
36 | rnd_init := func(n int) []byte {
37 | b := make([]byte, n)
38 | for i := range b {
39 | b[i] = letterRunes[rand_b(len(letterRunes))]
40 | }
41 | return b
42 | }
43 | randomStr = rnd_init(100000)
44 | rnd := func(n int) []byte {
45 | start := rand.Intn(1000)
46 | return randomStr[start : start+n]
47 | }
48 |
49 | runtime.GOMAXPROCS(1)
50 |
51 | bsm := Make(8, 40000, 100)
52 |
53 | single_pass := func(disp bool) {
54 | alloc := bsm.MakeAllocator()
55 |
56 | els_ok, els_fail, els_len := 0, 0, 0
57 | stored := make([][]byte, 0, 0)
58 | n_strings := rand.Intn(200) + 10
59 |
60 | for i := 0; i < n_strings; i++ {
61 | n := 0
62 | if i%10 == 0 {
63 | n = rand.Intn(60000) + 30000
64 | } else {
65 | n = rand.Intn(600)
66 | }
67 |
68 | tostore := rnd(n)
69 | els_len += n
70 | stored = append(stored, tostore)
71 | if i%2 == 0 {
72 | // store test1
73 | //_sa := make([]byte, n)
74 | _sa := alloc.Allocate(n)[0:n]
75 | copy(_sa, tostore)
76 | stored = append(stored, _sa)
77 | } else {
78 | //_sa := make([]byte, 0, n)
79 | _sa := alloc.Allocate(n)
80 | _sa_b := _sa[0:n]
81 | for _, b := range tostore {
82 | _sa = append(_sa, b)
83 | }
84 | stored = append(stored, _sa_b)
85 | }
86 |
87 | for i := 0; i < len(stored); i += 2 {
88 | ok := bytes.Compare(stored[i], stored[i+1]) == 0
89 | if ok {
90 | els_ok++
91 | } else {
92 | els_fail++
93 | fmt.Println("!! ", len(stored[i]), len(stored[i+1]))
94 | }
95 | }
96 | }
97 | alloc.Release()
98 | if disp {
99 | fmt.Println(n_strings, "Strings... ", "OK: ", els_ok, " Fail", els_fail, " Bytes: ", els_len)
100 | }
101 |
102 | }
103 |
104 | threads := 10
105 | for z := 0; z < 60/threads; z++ {
106 | wg := sync.WaitGroup{}
107 | for i := 0; i < threads; i++ {
108 | i := i
109 | wg.Add(1)
110 | go func() {
111 | fmt.Println("Started : ", i)
112 | for i := 0; i < 300; i++ {
113 | single_pass(i%60 == 0)
114 | }
115 | wg.Done()
116 | }()
117 | }
118 | wg.Wait()
119 | }
120 |
121 | fmt.Println(float64(time.Now().UnixMilli()-ts) / 1000)
122 |
123 | var garC debug.GCStats
124 | debug.ReadGCStats(&garC)
125 | fmt.Printf("\nLastGC:\t%s", garC.LastGC) // time of last collection
126 | fmt.Printf("\nNumGC:\t%d", garC.NumGC) // number of garbage collections
127 | fmt.Printf("\nPauseTotal:\t%s", garC.PauseTotal) // total pause for all collections
128 |
129 | }
130 |
131 | func BenchmarkByteslabs(b *testing.B) {
132 |
133 | ts := time.Now().UnixMilli()
134 | runtime.GOMAXPROCS(60)
135 |
136 | bsm := Make(8, 40000, 100)
137 | single_pass := func(disp bool) {
138 | alloc := bsm.MakeAllocator()
139 | n_strings := rand_b(200) + 10
140 |
141 | for i := 0; i < n_strings; i++ {
142 | n := 0
143 | if i%10 == 0 {
144 | n = rand_b(60000) + 30000
145 | } else {
146 | n = rand_b(600)
147 | }
148 | alloc.Allocate(n)
149 | }
150 | alloc.Release()
151 | }
152 |
153 | threads := 60
154 | fmt.Println("Startig threads: ")
155 | for z := 0; z < 60/threads; z++ {
156 | wg := sync.WaitGroup{}
157 | for i := 0; i < threads; i++ {
158 | i := i
159 | wg.Add(1)
160 | go func() {
161 | fmt.Print(i, " ")
162 | for i := 0; i < 1000; i++ {
163 | single_pass(i%60 == 0)
164 | }
165 | wg.Done()
166 | }()
167 | }
168 | wg.Wait()
169 | }
170 | fmt.Println()
171 |
172 | fmt.Println(float64(time.Now().UnixMilli()-ts) / 1000)
173 |
174 | var garC debug.GCStats
175 | debug.ReadGCStats(&garC)
176 | fmt.Printf("\nLastGC:\t%s", garC.LastGC) // time of last collection
177 | fmt.Printf("\nNumGC:\t%d", garC.NumGC) // number of garbage collections
178 | fmt.Printf("\nPauseTotal:\t%s", garC.PauseTotal) // total pause for all collections
179 |
180 | b.ReportAllocs()
181 | fmt.Println(bsm.GetStatusStr())
182 | }
183 |
--------------------------------------------------------------------------------
/handler_socket2/byteslabs2/status.go:
--------------------------------------------------------------------------------
1 | package byteslabs2
2 |
3 | import (
4 | "fmt"
5 | "github.com/slawomir-pryczek/HSServer/handler_socket2/hscommon"
6 | "strconv"
7 | "strings"
8 | )
9 |
10 | func (this *BSManager) GetStatus() (string, string) {
11 |
12 | ret := ""
13 | ret += fmt.Sprintf("Slab Allocator. Slab Size: %d x %d Slabs = %d items per Page. %d Pages.\n", this.conf_slab_size, this.conf_slab_count, this.conf_slab_size*this.conf_slab_count, len(this.mem_chunks))
14 | ret += "Alloc Full - We're allocating >1 SLABS, happens at page begin\n"
15 | ret += "Alloc Full Small - We're allocating 1 SLAB, happens at page's end to prevent fragmentation\n"
16 | ret += "Tail - We can put the data into already allocated SLAB's tail\n"
17 | ret += "OOM - Page is out of memory, we're using system allocator instead\n"
18 | ret += "Routed - Allocations was routed to least used slab page, allocations per routing\n"
19 | ret += "Routed Alloc - Allocations in routed slabs\n"
20 | ret += ". - Slab is EMPTY; F - Slab is FULL\n"
21 | ret += " "
22 |
23 | ret += ""
29 |
30 | tg := hscommon.NewTableGen("#", "Alloc Full", "Full Small", "Tail",
31 | "OOM ", "Slabs", "Routed ", "Routed Alloc")
32 | tg.SetClass("tab salloc")
33 |
34 | for k, chunk := range this.mem_chunks {
35 | chunk.mu.Lock()
36 |
37 | _slabs := ""
38 | for _, used := range chunk.slab_used {
39 | if used {
40 | _slabs += "F"
41 | } else {
42 | _slabs += "."
43 | }
44 | }
45 |
46 | percent_oom := int(chunk.stat_alloc_full + chunk.stat_alloc_full_small + chunk.stat_alloc_tail)
47 | if percent_oom > 0 {
48 | percent_oom = int((float64(chunk.stat_oom) / float64(percent_oom)) * 1000)
49 | }
50 | oom := fmt.Sprintf("%d (%d.%d%%)", chunk.stat_oom, percent_oom/10, percent_oom%10)
51 |
52 | apr := 0
53 | if chunk.stat_routed > 0 {
54 | apr = chunk.stat_routed_alloc * 10 / chunk.stat_routed
55 | }
56 | routed := fmt.Sprintf("%d (%d.%d apr)", chunk.stat_routed, apr/10, apr%10)
57 |
58 | tg.AddRow(strconv.Itoa(k), strconv.Itoa(chunk.stat_alloc_full), strconv.Itoa(chunk.stat_alloc_full_small),
59 | strconv.Itoa(chunk.stat_alloc_tail), oom, ""+_slabs+" ", routed, strconv.Itoa(chunk.stat_routed_alloc))
60 |
61 | chunk.mu.Unlock()
62 | }
63 |
64 | return "Slab Allocator \\ QCompress", ret + tg.Render()
65 | }
66 |
67 | func (this *BSManager) GetStatusStr() string {
68 | ret := make([]string, 0, 40)
69 | ret = append(ret, "=====")
70 | total_failed := 0
71 | total_f, total_fs, total_t := 0, 0, 0
72 | for k, v := range this.mem_chunks {
73 | ret = append(ret, fmt.Sprint(k, "Full:", v.stat_alloc_full, "Full Small:", v.stat_alloc_full_small,
74 | "Tail:", v.stat_alloc_tail, "OOM:", v.stat_oom, "Routed:", v.stat_routed,
75 | "Slab taken:", v.used_slab_count))
76 |
77 | total_failed += v.stat_oom - v.stat_routed
78 | total_f += v.stat_alloc_full
79 | total_fs += v.stat_alloc_full_small
80 | total_t += v.stat_alloc_tail
81 | }
82 |
83 | ret = append(ret, fmt.Sprintf("=Totals ... Full: %d Full Small: %d Tail: %d \n", total_f, total_fs, total_t))
84 | ret = append(ret, fmt.Sprintf("=Items Total: %d, Failed: %d\n",
85 | total_f+total_fs+total_t+total_failed, total_failed))
86 | ret = append(ret, fmt.Sprint("Real Total: ", ttT_total))
87 | return strings.Join(ret, "\n")
88 | }
89 |
--------------------------------------------------------------------------------
/handler_socket2/compress/engine.go:
--------------------------------------------------------------------------------
1 | package compress
2 |
3 | import (
4 | "encoding/binary"
5 | "fmt"
6 | )
7 |
8 | type Compressor struct {
9 | compressor chan piece_in
10 | compressor_fast chan piece_in
11 | c cpr
12 | }
13 |
14 | type piece_in struct {
15 | data_in []byte
16 | data_out []byte
17 | piece_num int
18 |
19 | ch_out chan<- piece_out
20 | }
21 |
22 | type piece_out struct {
23 | data_out []byte
24 | piece int
25 | }
26 |
27 | type cpr interface {
28 | compression_thread(c_in chan piece_in, symbol byte) bool
29 | block_size() int
30 | uncompress_simple(in []byte, size int) []byte
31 | get_status() string
32 | get_id() string
33 | }
34 |
35 | func CreateCompressor(threads int, c cpr) *Compressor {
36 |
37 | ret := Compressor{}
38 | ret.compressor = make(chan piece_in, 500)
39 | ret.compressor_fast = make(chan piece_in, 50)
40 | ret.c = c
41 |
42 | // create normal threads for doing multipart compression
43 | for i := 0; i < threads; i++ {
44 | go c.compression_thread(ret.compressor, 'N')
45 | }
46 |
47 | // create fastpath threads
48 | threads = threads / 2
49 | if threads < 6 {
50 | threads = 6
51 | }
52 | for i := 0; i < threads; i++ {
53 | go c.compression_thread(ret.compressor_fast, 'F')
54 | }
55 |
56 | return &ret
57 | }
58 |
59 | // this function will run fast compression
60 | func (this *Compressor) _fastpath(in []byte, out []byte) []byte {
61 | offset := 8
62 |
63 | // compress data
64 | ch_out := make(chan piece_out, 1)
65 | this.compressor_fast <- piece_in{in, out[offset:], 0, ch_out}
66 |
67 | // get response
68 | ret := <-ch_out
69 | if ret.data_out == nil || ret.piece < 0 || float32(len(ret.data_out)) > float32(len(in))*0.8 {
70 | return nil
71 | }
72 |
73 | out = out[0 : len(ret.data_out)+offset]
74 | binary.LittleEndian.PutUint32(out[0:4], uint32(len(ret.data_out)))
75 | binary.LittleEndian.PutUint32(out[4:8], 0)
76 | return out
77 | }
78 |
79 | func (this *Compressor) Compress(in []byte, out []byte) []byte {
80 |
81 | block_size := this.c.block_size()
82 | var in_progress = 0
83 | var in_len = len(in)
84 |
85 | if in_len < block_size {
86 | return this._fastpath(in, out)
87 | }
88 |
89 | is_broken := false
90 | out_size_total := 0
91 | tasks_no := (in_len / block_size)
92 | if in_len > tasks_no*block_size {
93 | tasks_no++
94 | }
95 |
96 | // function to get data back from workers
97 | chunks := make([]int, tasks_no+1)
98 | ch_out := make(chan piece_out, 8)
99 | get_next_piece := func() {
100 | ret := <-ch_out
101 | in_progress--
102 |
103 | out_size_total += len(ret.data_out)
104 | is_broken = is_broken || ret.piece < 0
105 | if is_broken {
106 | return
107 | }
108 | chunks[ret.piece] = len(ret.data_out)
109 | }
110 |
111 | piece_no := 0
112 | for i := 0; i < in_len; i += block_size {
113 | end := i + block_size
114 | if end > in_len {
115 | end = in_len
116 | }
117 |
118 | this.compressor <- piece_in{in[i:end], out[i:end], piece_no, ch_out}
119 | piece_no++
120 | in_progress++
121 |
122 | if in_progress >= 12 {
123 | get_next_piece()
124 | if is_broken {
125 | break
126 | }
127 | }
128 | }
129 |
130 | for in_progress > 0 {
131 | get_next_piece()
132 | }
133 |
134 | if is_broken || float32(out_size_total) > float32(len(in))*0.8 {
135 | return nil
136 | }
137 |
138 | // merge the blocks together... skip space needed for allocation table first
139 | dst_pos := 4 * len(chunks)
140 | for i := 0; i < tasks_no; i++ {
141 |
142 | chunk_size := chunks[i]
143 | src_start := block_size * i
144 | copy(out[dst_pos:dst_pos+chunk_size], out[src_start:src_start+chunk_size])
145 |
146 | dst_pos += chunk_size
147 | }
148 |
149 | for i := 0; i < len(chunks); i++ {
150 | pos := i * 4
151 | binary.LittleEndian.PutUint32(out[pos:pos+4], uint32(chunks[i]))
152 | }
153 |
154 | //fmt.Println("Yx", len(in), ">", dst_pos, "|", chunks, out[0:28], out[dst_pos-16:dst_pos])
155 | //fmt.Println(dst_pos, out[0:109])
156 | return out[0:dst_pos]
157 | }
158 |
159 | func (this *Compressor) Uncompress(in []byte) []byte {
160 |
161 | chunks := make([]uint32, 0, 10)
162 | pos := 0
163 | for pos < len(in)-4 {
164 | size := binary.LittleEndian.Uint32(in[pos : pos+4])
165 | pos += 4
166 |
167 | if size == 0 {
168 | break
169 | }
170 | chunks = append(chunks, size)
171 | }
172 | if len(chunks) == 0 {
173 | return nil
174 | }
175 |
176 | out := make([]byte, 0, len(in)*2)
177 | chunk := 0
178 | for pos < len(in) && chunk < len(chunks) {
179 |
180 | chunk_size := int(chunks[chunk])
181 | chunk_content := this.c.uncompress_simple(in[pos:pos+chunk_size], chunk_size)
182 |
183 | fmt.Println("UNC c", chunk, len(in[pos:pos+chunk_size]), in[pos:pos+10])
184 |
185 | chunk++
186 | pos += chunk_size
187 | out = append(out, chunk_content...)
188 | }
189 |
190 | return out
191 | }
192 |
193 | func (this *Compressor) GetStatus() string {
194 | return this.c.get_status()
195 | }
196 |
197 | func (this *Compressor) GetID() string {
198 | return this.c.get_id()
199 | }
200 |
--------------------------------------------------------------------------------
/handler_socket2/compress/flate.go:
--------------------------------------------------------------------------------
1 | package compress
2 |
3 | import (
4 | "bytes"
5 | "compress/flate"
6 | "io"
7 | )
8 |
9 | type compressionflate struct {
10 | stat *stat
11 | }
12 |
13 | func MakeFlate() *compressionflate {
14 | ret := compressionflate{}
15 | ret.stat = MakeStat("Flate")
16 | return &ret
17 | }
18 |
19 | func (c *compressionflate) compression_thread(c_in chan piece_in, symbol byte) bool {
20 |
21 | block_size := c.block_size()
22 |
23 | buf := make([]byte, 0, block_size+800)
24 | buffer := bytes.NewBuffer(buf)
25 | writer, err := flate.NewWriter(buffer, flate.BestSpeed)
26 | if err != nil {
27 | return false
28 | }
29 |
30 | thread_id := c.stat.ThreadAdd(symbol)
31 | for {
32 | buffer = bytes.NewBuffer(buf[0:0:cap(buf)])
33 | writer.Reset(buffer)
34 |
35 | data := <-c_in
36 | c.stat.ThreadSetRunning(thread_id, true)
37 |
38 | io.Copy(writer, bytes.NewReader(data.data_in))
39 | writer.Close()
40 |
41 | b := buffer.Bytes()
42 | if len(b) >= len(data.data_in) && len(b) > block_size/3 {
43 | data.ch_out <- piece_out{nil, -1}
44 | c.stat.reportCompressionRatioTooLow()
45 | continue
46 | }
47 | if len(b) > len(data.data_out) {
48 | data.ch_out <- piece_out{nil, -1}
49 | c.stat.reportBufferTooSmall()
50 | continue
51 | }
52 |
53 | stat_in := len(data.data_in)
54 | stat_out := len(b)
55 | copy(data.data_out[0:len(b)], b)
56 | data.ch_out <- piece_out{data.data_out[0:len(b)], data.piece_num}
57 |
58 | c.stat.doStats(thread_id, data.piece_num, stat_in, stat_out)
59 | }
60 | return true
61 | }
62 |
63 | func (c *compressionflate) uncompress_simple(in []byte, size int) []byte {
64 |
65 | reader := flate.NewReader(bytes.NewReader(in))
66 | buf := make([]byte, 0, len(in)*3)
67 | buffer := bytes.NewBuffer(buf)
68 | _, err := io.Copy(buffer, reader)
69 | if err != nil {
70 | return nil
71 | }
72 |
73 | return buffer.Bytes()
74 | }
75 |
76 | func (c *compressionflate) get_status() string {
77 | return c.stat.GetStatus()
78 | }
79 |
80 | func (c compressionflate) block_size() int {
81 | return 60000
82 | }
83 |
84 | func (c *compressionflate) get_id() string {
85 | return "mp-flate"
86 | }
87 |
--------------------------------------------------------------------------------
/handler_socket2/compress/simple.go:
--------------------------------------------------------------------------------
1 | package compress
2 |
3 | import (
4 | "bytes"
5 | "compress/flate"
6 | "fmt"
7 | "sync"
8 |
9 | "github.com/slawomir-pryczek/HSServer/handler_socket2/byteslabs"
10 | )
11 |
12 | const max_compressors = 20
13 |
14 | var mu sync.Mutex
15 | var f_writers []*flate.Writer
16 | var fw_stat_compressors_reuse = 0
17 | var fw_stat_compressors_created = 0
18 | var fw_stat_underflows = 0
19 | var fw_stat_overflows = 0
20 | var fw_stat_underflows_b = 0
21 | var fw_stat_overflows_b = 0
22 |
23 | func init() {
24 | f_writers = make([]*flate.Writer, max_compressors)
25 | for i := 0; i < max_compressors; i++ {
26 | tmp, _ := flate.NewWriter(nil, 2)
27 | f_writers[i] = tmp
28 | }
29 | }
30 |
31 | func CompressSimple(data []byte, alloc *byteslabs.Allocator) []byte {
32 |
33 | mu.Lock()
34 | var writer *flate.Writer = nil
35 | if len(f_writers) > 0 {
36 | writer = f_writers[len(f_writers)-1]
37 | f_writers = f_writers[:len(f_writers)-1]
38 | fw_stat_compressors_reuse++
39 | } else {
40 | fw_stat_compressors_created++
41 | }
42 | mu.Unlock()
43 | if writer == nil {
44 | writer, _ = flate.NewWriter(nil, 2)
45 | }
46 |
47 | _d_len := len(data)
48 | _d_len = int(float32(_d_len) / 1.3)
49 |
50 | b := bytes.NewBuffer(alloc.Allocate(_d_len))
51 | writer.Reset(b)
52 | writer.Write(data)
53 | writer.Close()
54 |
55 | diff := _d_len - b.Len()
56 | if diff >= 0 {
57 | fw_stat_underflows++
58 | fw_stat_underflows_b += diff
59 | } else {
60 | fw_stat_overflows++
61 | fw_stat_overflows_b += -diff
62 | }
63 |
64 | mu.Lock()
65 | f_writers = append(f_writers, writer)
66 | mu.Unlock()
67 |
68 | return b.Bytes()
69 | }
70 |
71 | func CompressSimpleStatus() string {
72 |
73 | ret := ""
74 |
75 | mu.Lock()
76 | defer mu.Unlock()
77 |
78 | ret += fmt.Sprintf("FastCompress*: %d, SlowCompress**: %d PreAllocated Compressors: %d\n",
79 | fw_stat_compressors_reuse, fw_stat_compressors_created, max_compressors)
80 | ret += " * Compressors that are re-used without memory re-allocation, ** Compressors fully re-allocated\n"
81 |
82 | _u := float64(fw_stat_underflows_b) / float64(fw_stat_underflows)
83 | _o := float64(fw_stat_overflows_b) / float64(fw_stat_overflows)
84 | if fw_stat_underflows == 0 {
85 | _u = 0
86 | }
87 | if fw_stat_overflows == 0 {
88 | _o = 0
89 | }
90 | ret += fmt.Sprintf("Buffer Predictions - Underflows: %d / %.1fKB Underflow Per Request\n", fw_stat_underflows, _u/float64(1024.0))
91 | ret += fmt.Sprintf("Buffer Predictions - Overflows: %d / %.1fKB Overflow Per Request\n", fw_stat_overflows, _o/float64(1024.0))
92 | ret += " Underflows mean that we allocated too large buffer from SLAB, this is normal situation.\n"
93 | ret += " Overflow means that we needed to re-allocate compression buffer because there was too little space\n"
94 |
95 | return ret
96 | }
97 |
--------------------------------------------------------------------------------
/handler_socket2/compress/snappy.go:
--------------------------------------------------------------------------------
1 | package compress
2 |
3 | import (
4 | "github.com/slawomir-pryczek/HSServer/handler_socket2/compress/snappy"
5 | )
6 |
7 | type compressionsnappy struct {
8 | stat *stat
9 | }
10 |
11 | func MakeSnappy() *compressionsnappy {
12 | ret := compressionsnappy{}
13 | ret.stat = MakeStat("Snappy")
14 | return &ret
15 | }
16 |
17 | func (c *compressionsnappy) compression_thread(c_in chan piece_in, symbol byte) bool {
18 |
19 | block_size := c.block_size()
20 |
21 | thread_id := c.stat.ThreadAdd(symbol)
22 | buf := make([]byte, 0, block_size+800)
23 | for {
24 | data := <-c_in
25 | c.stat.ThreadSetRunning(thread_id, true)
26 | b := snappy.Encode(buf, data.data_in)
27 |
28 | if len(b) >= len(data.data_in) && len(b) > block_size/3 {
29 | data.ch_out <- piece_out{nil, -1}
30 | c.stat.reportCompressionRatioTooLow()
31 | continue
32 | }
33 | if len(b) > len(data.data_out) {
34 | data.ch_out <- piece_out{nil, -1}
35 | c.stat.reportBufferTooSmall()
36 | continue
37 | }
38 |
39 | stat_in := len(data.data_in)
40 | stat_out := len(b)
41 | copy(data.data_out[0:len(b)], b)
42 | data.ch_out <- piece_out{data.data_out[0:len(b)], data.piece_num}
43 |
44 | c.stat.doStats(thread_id, data.piece_num, stat_in, stat_out)
45 | }
46 | return true
47 | }
48 |
49 | func (c *compressionsnappy) uncompress_simple(in []byte, size int) []byte {
50 |
51 | d_len, err := snappy.DecodedLen(in)
52 | if err != nil {
53 | return nil
54 | }
55 | buf := make([]byte, 0, d_len)
56 |
57 | ret, err1 := snappy.Decode(buf, in)
58 | if err1 != nil {
59 | return nil
60 | }
61 | return ret
62 | }
63 |
64 | func (c *compressionsnappy) get_status() string {
65 | return c.stat.GetStatus()
66 | }
67 |
68 | func (c compressionsnappy) block_size() int {
69 | return 120000
70 | }
71 |
72 | func (c *compressionsnappy) get_id() string {
73 | return "mp-snappy"
74 | }
75 |
--------------------------------------------------------------------------------
/handler_socket2/compress/snappy/.gitignore:
--------------------------------------------------------------------------------
1 | cmd/snappytool/snappytool
2 | testdata/bench
3 |
4 | # These explicitly listed benchmark data files are for an obsolete version of
5 | # snappy_test.go.
6 | testdata/alice29.txt
7 | testdata/asyoulik.txt
8 | testdata/fireworks.jpeg
9 | testdata/geo.protodata
10 | testdata/html
11 | testdata/html_x_4
12 | testdata/kppkn.gtb
13 | testdata/lcet10.txt
14 | testdata/paper-100k.pdf
15 | testdata/plrabn12.txt
16 | testdata/urls.10K
17 |
--------------------------------------------------------------------------------
/handler_socket2/compress/snappy/AUTHORS:
--------------------------------------------------------------------------------
1 | # This is the official list of Snappy-Go authors for copyright purposes.
2 | # This file is distinct from the CONTRIBUTORS files.
3 | # See the latter for an explanation.
4 |
5 | # Names should be added to this file as
6 | # Name or Organization
7 | # The email address is not required for organizations.
8 |
9 | # Please keep the list sorted.
10 |
11 | Damian Gryski
12 | Google Inc.
13 | Jan Mercl <0xjnml@gmail.com>
14 | Klaus Post
15 | Rodolfo Carvalho
16 | Sebastien Binet
17 |
--------------------------------------------------------------------------------
/handler_socket2/compress/snappy/CONTRIBUTORS:
--------------------------------------------------------------------------------
1 | # This is the official list of people who can contribute
2 | # (and typically have contributed) code to the Snappy-Go repository.
3 | # The AUTHORS file lists the copyright holders; this file
4 | # lists people. For example, Google employees are listed here
5 | # but not in AUTHORS, because Google holds the copyright.
6 | #
7 | # The submission process automatically checks to make sure
8 | # that people submitting code are listed in this file (by email address).
9 | #
10 | # Names should be added to this file only after verifying that
11 | # the individual or the individual's organization has agreed to
12 | # the appropriate Contributor License Agreement, found here:
13 | #
14 | # http://code.google.com/legal/individual-cla-v1.0.html
15 | # http://code.google.com/legal/corporate-cla-v1.0.html
16 | #
17 | # The agreement for individuals can be filled out on the web.
18 | #
19 | # When adding J Random Contributor's name to this file,
20 | # either J's name or J's organization's name should be
21 | # added to the AUTHORS file, depending on whether the
22 | # individual or corporate CLA was used.
23 |
24 | # Names should be added to this file like so:
25 | # Name
26 |
27 | # Please keep the list sorted.
28 |
29 | Damian Gryski
30 | Jan Mercl <0xjnml@gmail.com>
31 | Kai Backman
32 | Klaus Post
33 | Marc-Antoine Ruel
34 | Nigel Tao
35 | Rob Pike
36 | Rodolfo Carvalho
37 | Russ Cox
38 | Sebastien Binet
39 |
--------------------------------------------------------------------------------
/handler_socket2/compress/snappy/LICENSE:
--------------------------------------------------------------------------------
1 | Copyright (c) 2011 The Snappy-Go Authors. All rights reserved.
2 |
3 | Redistribution and use in source and binary forms, with or without
4 | modification, are permitted provided that the following conditions are
5 | met:
6 |
7 | * Redistributions of source code must retain the above copyright
8 | notice, this list of conditions and the following disclaimer.
9 | * Redistributions in binary form must reproduce the above
10 | copyright notice, this list of conditions and the following disclaimer
11 | in the documentation and/or other materials provided with the
12 | distribution.
13 | * Neither the name of Google Inc. nor the names of its
14 | contributors may be used to endorse or promote products derived from
15 | this software without specific prior written permission.
16 |
17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 |
--------------------------------------------------------------------------------
/handler_socket2/compress/snappy/README:
--------------------------------------------------------------------------------
1 | The Snappy compression format in the Go programming language.
2 |
3 | To download and install from source:
4 | $ go get github.com/golang/snappy
5 |
6 | Unless otherwise noted, the Snappy-Go source files are distributed
7 | under the BSD-style license found in the LICENSE file.
8 |
9 |
10 |
11 | Benchmarks.
12 |
13 | The golang/snappy benchmarks include compressing (Z) and decompressing (U) ten
14 | or so files, the same set used by the C++ Snappy code (github.com/google/snappy
15 | and note the "google", not "golang"). On an "Intel(R) Core(TM) i7-3770 CPU @
16 | 3.40GHz", Go's GOARCH=amd64 numbers as of 2016-05-29:
17 |
18 | "go test -test.bench=."
19 |
20 | _UFlat0-8 2.19GB/s ± 0% html
21 | _UFlat1-8 1.41GB/s ± 0% urls
22 | _UFlat2-8 23.5GB/s ± 2% jpg
23 | _UFlat3-8 1.91GB/s ± 0% jpg_200
24 | _UFlat4-8 14.0GB/s ± 1% pdf
25 | _UFlat5-8 1.97GB/s ± 0% html4
26 | _UFlat6-8 814MB/s ± 0% txt1
27 | _UFlat7-8 785MB/s ± 0% txt2
28 | _UFlat8-8 857MB/s ± 0% txt3
29 | _UFlat9-8 719MB/s ± 1% txt4
30 | _UFlat10-8 2.84GB/s ± 0% pb
31 | _UFlat11-8 1.05GB/s ± 0% gaviota
32 |
33 | _ZFlat0-8 1.04GB/s ± 0% html
34 | _ZFlat1-8 534MB/s ± 0% urls
35 | _ZFlat2-8 15.7GB/s ± 1% jpg
36 | _ZFlat3-8 740MB/s ± 3% jpg_200
37 | _ZFlat4-8 9.20GB/s ± 1% pdf
38 | _ZFlat5-8 991MB/s ± 0% html4
39 | _ZFlat6-8 379MB/s ± 0% txt1
40 | _ZFlat7-8 352MB/s ± 0% txt2
41 | _ZFlat8-8 396MB/s ± 1% txt3
42 | _ZFlat9-8 327MB/s ± 1% txt4
43 | _ZFlat10-8 1.33GB/s ± 1% pb
44 | _ZFlat11-8 605MB/s ± 1% gaviota
45 |
46 |
47 |
48 | "go test -test.bench=. -tags=noasm"
49 |
50 | _UFlat0-8 621MB/s ± 2% html
51 | _UFlat1-8 494MB/s ± 1% urls
52 | _UFlat2-8 23.2GB/s ± 1% jpg
53 | _UFlat3-8 1.12GB/s ± 1% jpg_200
54 | _UFlat4-8 4.35GB/s ± 1% pdf
55 | _UFlat5-8 609MB/s ± 0% html4
56 | _UFlat6-8 296MB/s ± 0% txt1
57 | _UFlat7-8 288MB/s ± 0% txt2
58 | _UFlat8-8 309MB/s ± 1% txt3
59 | _UFlat9-8 280MB/s ± 1% txt4
60 | _UFlat10-8 753MB/s ± 0% pb
61 | _UFlat11-8 400MB/s ± 0% gaviota
62 |
63 | _ZFlat0-8 409MB/s ± 1% html
64 | _ZFlat1-8 250MB/s ± 1% urls
65 | _ZFlat2-8 12.3GB/s ± 1% jpg
66 | _ZFlat3-8 132MB/s ± 0% jpg_200
67 | _ZFlat4-8 2.92GB/s ± 0% pdf
68 | _ZFlat5-8 405MB/s ± 1% html4
69 | _ZFlat6-8 179MB/s ± 1% txt1
70 | _ZFlat7-8 170MB/s ± 1% txt2
71 | _ZFlat8-8 189MB/s ± 1% txt3
72 | _ZFlat9-8 164MB/s ± 1% txt4
73 | _ZFlat10-8 479MB/s ± 1% pb
74 | _ZFlat11-8 270MB/s ± 1% gaviota
75 |
76 |
77 |
78 | For comparison (Go's encoded output is byte-for-byte identical to C++'s), here
79 | are the numbers from C++ Snappy's
80 |
81 | make CXXFLAGS="-O2 -DNDEBUG -g" clean snappy_unittest.log && cat snappy_unittest.log
82 |
83 | BM_UFlat/0 2.4GB/s html
84 | BM_UFlat/1 1.4GB/s urls
85 | BM_UFlat/2 21.8GB/s jpg
86 | BM_UFlat/3 1.5GB/s jpg_200
87 | BM_UFlat/4 13.3GB/s pdf
88 | BM_UFlat/5 2.1GB/s html4
89 | BM_UFlat/6 1.0GB/s txt1
90 | BM_UFlat/7 959.4MB/s txt2
91 | BM_UFlat/8 1.0GB/s txt3
92 | BM_UFlat/9 864.5MB/s txt4
93 | BM_UFlat/10 2.9GB/s pb
94 | BM_UFlat/11 1.2GB/s gaviota
95 |
96 | BM_ZFlat/0 944.3MB/s html (22.31 %)
97 | BM_ZFlat/1 501.6MB/s urls (47.78 %)
98 | BM_ZFlat/2 14.3GB/s jpg (99.95 %)
99 | BM_ZFlat/3 538.3MB/s jpg_200 (73.00 %)
100 | BM_ZFlat/4 8.3GB/s pdf (83.30 %)
101 | BM_ZFlat/5 903.5MB/s html4 (22.52 %)
102 | BM_ZFlat/6 336.0MB/s txt1 (57.88 %)
103 | BM_ZFlat/7 312.3MB/s txt2 (61.91 %)
104 | BM_ZFlat/8 353.1MB/s txt3 (54.99 %)
105 | BM_ZFlat/9 289.9MB/s txt4 (66.26 %)
106 | BM_ZFlat/10 1.2GB/s pb (19.68 %)
107 | BM_ZFlat/11 527.4MB/s gaviota (37.72 %)
108 |
--------------------------------------------------------------------------------
/handler_socket2/compress/snappy/cmd/snappytool/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "errors"
5 | "flag"
6 | "io/ioutil"
7 | "os"
8 |
9 | "github.com/golang/snappy"
10 | )
11 |
12 | var (
13 | decode = flag.Bool("d", false, "decode")
14 | encode = flag.Bool("e", false, "encode")
15 | )
16 |
17 | func run() error {
18 | flag.Parse()
19 | if *decode == *encode {
20 | return errors.New("exactly one of -d or -e must be given")
21 | }
22 |
23 | in, err := ioutil.ReadAll(os.Stdin)
24 | if err != nil {
25 | return err
26 | }
27 |
28 | out := []byte(nil)
29 | if *decode {
30 | out, err = snappy.Decode(nil, in)
31 | if err != nil {
32 | return err
33 | }
34 | } else {
35 | out = snappy.Encode(nil, in)
36 | }
37 | _, err = os.Stdout.Write(out)
38 | return err
39 | }
40 |
41 | func main() {
42 | if err := run(); err != nil {
43 | os.Stderr.WriteString(err.Error() + "\n")
44 | os.Exit(1)
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/handler_socket2/compress/snappy/decode_amd64.go:
--------------------------------------------------------------------------------
1 | // Copyright 2016 The Snappy-Go Authors. All rights reserved.
2 | // Use of this source code is governed by a BSD-style
3 | // license that can be found in the LICENSE file.
4 |
5 | // +build !appengine
6 | // +build gc
7 | // +build !noasm
8 |
9 | package snappy
10 |
11 | // decode has the same semantics as in decode_other.go.
12 | //
13 | //go:noescape
14 | func decode(dst, src []byte) int
15 |
--------------------------------------------------------------------------------
/handler_socket2/compress/snappy/decode_other.go:
--------------------------------------------------------------------------------
1 | // Copyright 2016 The Snappy-Go Authors. All rights reserved.
2 | // Use of this source code is governed by a BSD-style
3 | // license that can be found in the LICENSE file.
4 |
5 | // +build !amd64 appengine !gc noasm
6 |
7 | package snappy
8 |
9 | // decode writes the decoding of src to dst. It assumes that the varint-encoded
10 | // length of the decompressed bytes has already been read, and that len(dst)
11 | // equals that length.
12 | //
13 | // It returns 0 on success or a decodeErrCodeXxx error code on failure.
14 | func decode(dst, src []byte) int {
15 | var d, s, offset, length int
16 | for s < len(src) {
17 | switch src[s] & 0x03 {
18 | case tagLiteral:
19 | x := uint32(src[s] >> 2)
20 | switch {
21 | case x < 60:
22 | s++
23 | case x == 60:
24 | s += 2
25 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
26 | return decodeErrCodeCorrupt
27 | }
28 | x = uint32(src[s-1])
29 | case x == 61:
30 | s += 3
31 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
32 | return decodeErrCodeCorrupt
33 | }
34 | x = uint32(src[s-2]) | uint32(src[s-1])<<8
35 | case x == 62:
36 | s += 4
37 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
38 | return decodeErrCodeCorrupt
39 | }
40 | x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16
41 | case x == 63:
42 | s += 5
43 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
44 | return decodeErrCodeCorrupt
45 | }
46 | x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24
47 | }
48 | length = int(x) + 1
49 | if length <= 0 {
50 | return decodeErrCodeUnsupportedLiteralLength
51 | }
52 | if length > len(dst)-d || length > len(src)-s {
53 | return decodeErrCodeCorrupt
54 | }
55 | copy(dst[d:], src[s:s+length])
56 | d += length
57 | s += length
58 | continue
59 |
60 | case tagCopy1:
61 | s += 2
62 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
63 | return decodeErrCodeCorrupt
64 | }
65 | length = 4 + int(src[s-2])>>2&0x7
66 | offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1]))
67 |
68 | case tagCopy2:
69 | s += 3
70 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
71 | return decodeErrCodeCorrupt
72 | }
73 | length = 1 + int(src[s-3])>>2
74 | offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8)
75 |
76 | case tagCopy4:
77 | s += 5
78 | if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line.
79 | return decodeErrCodeCorrupt
80 | }
81 | length = 1 + int(src[s-5])>>2
82 | offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24)
83 | }
84 |
85 | if offset <= 0 || d < offset || length > len(dst)-d {
86 | return decodeErrCodeCorrupt
87 | }
88 | // Copy from an earlier sub-slice of dst to a later sub-slice.
89 | // If no overlap, use the built-in copy:
90 | if offset >= length {
91 | copy(dst[d:d+length], dst[d-offset:])
92 | d += length
93 | continue
94 | }
95 |
96 | // Unlike the built-in copy function, this byte-by-byte copy always runs
97 | // forwards, even if the slices overlap. Conceptually, this is:
98 | //
99 | // d += forwardCopy(dst[d:d+length], dst[d-offset:])
100 | //
101 | // We align the slices into a and b and show the compiler they are the same size.
102 | // This allows the loop to run without bounds checks.
103 | a := dst[d : d+length]
104 | b := dst[d-offset:]
105 | b = b[:len(a)]
106 | for i := range a {
107 | a[i] = b[i]
108 | }
109 | d += length
110 | }
111 | if d != len(dst) {
112 | return decodeErrCodeCorrupt
113 | }
114 | return 0
115 | }
116 |
--------------------------------------------------------------------------------
/handler_socket2/compress/snappy/encode_amd64.go:
--------------------------------------------------------------------------------
1 | // Copyright 2016 The Snappy-Go Authors. All rights reserved.
2 | // Use of this source code is governed by a BSD-style
3 | // license that can be found in the LICENSE file.
4 |
5 | // +build !appengine
6 | // +build gc
7 | // +build !noasm
8 |
9 | package snappy
10 |
11 | // emitLiteral has the same semantics as in encode_other.go.
12 | //
13 | //go:noescape
14 | func emitLiteral(dst, lit []byte) int
15 |
16 | // emitCopy has the same semantics as in encode_other.go.
17 | //
18 | //go:noescape
19 | func emitCopy(dst []byte, offset, length int) int
20 |
21 | // extendMatch has the same semantics as in encode_other.go.
22 | //
23 | //go:noescape
24 | func extendMatch(src []byte, i, j int) int
25 |
26 | // encodeBlock has the same semantics as in encode_other.go.
27 | //
28 | //go:noescape
29 | func encodeBlock(dst, src []byte) (d int)
30 |
--------------------------------------------------------------------------------
/handler_socket2/compress/snappy/gox.mod:
--------------------------------------------------------------------------------
1 | module github.com/golang/snappy
2 |
--------------------------------------------------------------------------------
/handler_socket2/compress/snappy/misc/main.cpp:
--------------------------------------------------------------------------------
1 | /*
2 | This is a C version of the cmd/snappytool Go program.
3 |
4 | To build the snappytool binary:
5 | g++ main.cpp /usr/lib/libsnappy.a -o snappytool
6 | or, if you have built the C++ snappy library from source:
7 | g++ main.cpp /path/to/your/snappy/.libs/libsnappy.a -o snappytool
8 | after running "make" from your snappy checkout directory.
9 | */
10 |
11 | #include
12 | #include
13 | #include
14 | #include
15 |
16 | #include "snappy.h"
17 |
18 | #define N 1000000
19 |
20 | char dst[N];
21 | char src[N];
22 |
23 | int main(int argc, char** argv) {
24 | // Parse args.
25 | if (argc != 2) {
26 | fprintf(stderr, "exactly one of -d or -e must be given\n");
27 | return 1;
28 | }
29 | bool decode = strcmp(argv[1], "-d") == 0;
30 | bool encode = strcmp(argv[1], "-e") == 0;
31 | if (decode == encode) {
32 | fprintf(stderr, "exactly one of -d or -e must be given\n");
33 | return 1;
34 | }
35 |
36 | // Read all of stdin into src[:s].
37 | size_t s = 0;
38 | while (1) {
39 | if (s == N) {
40 | fprintf(stderr, "input too large\n");
41 | return 1;
42 | }
43 | ssize_t n = read(0, src+s, N-s);
44 | if (n == 0) {
45 | break;
46 | }
47 | if (n < 0) {
48 | fprintf(stderr, "read error: %s\n", strerror(errno));
49 | // TODO: handle EAGAIN, EINTR?
50 | return 1;
51 | }
52 | s += n;
53 | }
54 |
55 | // Encode or decode src[:s] to dst[:d], and write to stdout.
56 | size_t d = 0;
57 | if (encode) {
58 | if (N < snappy::MaxCompressedLength(s)) {
59 | fprintf(stderr, "input too large after encoding\n");
60 | return 1;
61 | }
62 | snappy::RawCompress(src, s, dst, &d);
63 | } else {
64 | if (!snappy::GetUncompressedLength(src, s, &d)) {
65 | fprintf(stderr, "could not get uncompressed length\n");
66 | return 1;
67 | }
68 | if (N < d) {
69 | fprintf(stderr, "input too large after decoding\n");
70 | return 1;
71 | }
72 | if (!snappy::RawUncompress(src, s, dst)) {
73 | fprintf(stderr, "input was not valid Snappy-compressed data\n");
74 | return 1;
75 | }
76 | }
77 | write(1, dst, d);
78 | return 0;
79 | }
80 |
--------------------------------------------------------------------------------
/handler_socket2/compress/snappy/snappy.go:
--------------------------------------------------------------------------------
1 | // Copyright 2011 The Snappy-Go Authors. All rights reserved.
2 | // Use of this source code is governed by a BSD-style
3 | // license that can be found in the LICENSE file.
4 |
5 | // Package snappy implements the Snappy compression format. It aims for very
6 | // high speeds and reasonable compression.
7 | //
8 | // There are actually two Snappy formats: block and stream. They are related,
9 | // but different: trying to decompress block-compressed data as a Snappy stream
10 | // will fail, and vice versa. The block format is the Decode and Encode
11 | // functions and the stream format is the Reader and Writer types.
12 | //
13 | // The block format, the more common case, is used when the complete size (the
14 | // number of bytes) of the original data is known upfront, at the time
15 | // compression starts. The stream format, also known as the framing format, is
16 | // for when that isn't always true.
17 | //
18 | // The canonical, C++ implementation is at https://github.com/google/snappy and
19 | // it only implements the block format.
20 | package snappy
21 |
22 | import (
23 | "hash/crc32"
24 | )
25 |
26 | /*
27 | Each encoded block begins with the varint-encoded length of the decoded data,
28 | followed by a sequence of chunks. Chunks begin and end on byte boundaries. The
29 | first byte of each chunk is broken into its 2 least and 6 most significant bits
30 | called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag.
31 | Zero means a literal tag. All other values mean a copy tag.
32 |
33 | For literal tags:
34 | - If m < 60, the next 1 + m bytes are literal bytes.
35 | - Otherwise, let n be the little-endian unsigned integer denoted by the next
36 | m - 59 bytes. The next 1 + n bytes after that are literal bytes.
37 |
38 | For copy tags, length bytes are copied from offset bytes ago, in the style of
39 | Lempel-Ziv compression algorithms. In particular:
40 | - For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12).
41 | The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10
42 | of the offset. The next byte is bits 0-7 of the offset.
43 | - For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65).
44 | The length is 1 + m. The offset is the little-endian unsigned integer
45 | denoted by the next 2 bytes.
46 | - For l == 3, this tag is a legacy format that is no longer issued by most
47 | encoders. Nonetheless, the offset ranges in [0, 1<<32) and the length in
48 | [1, 65). The length is 1 + m. The offset is the little-endian unsigned
49 | integer denoted by the next 4 bytes.
50 | */
51 | const (
52 | tagLiteral = 0x00
53 | tagCopy1 = 0x01
54 | tagCopy2 = 0x02
55 | tagCopy4 = 0x03
56 | )
57 |
58 | const (
59 | checksumSize = 4
60 | chunkHeaderSize = 4
61 | magicChunk = "\xff\x06\x00\x00" + magicBody
62 | magicBody = "sNaPpY"
63 |
64 | // maxBlockSize is the maximum size of the input to encodeBlock. It is not
65 | // part of the wire format per se, but some parts of the encoder assume
66 | // that an offset fits into a uint16.
67 | //
68 | // Also, for the framing format (Writer type instead of Encode function),
69 | // https://github.com/google/snappy/blob/master/framing_format.txt says
70 | // that "the uncompressed data in a chunk must be no longer than 65536
71 | // bytes".
72 | maxBlockSize = 65536
73 |
74 | // maxEncodedLenOfMaxBlockSize equals MaxEncodedLen(maxBlockSize), but is
75 | // hard coded to be a const instead of a variable, so that obufLen can also
76 | // be a const. Their equivalence is confirmed by
77 | // TestMaxEncodedLenOfMaxBlockSize.
78 | maxEncodedLenOfMaxBlockSize = 76490
79 |
80 | obufHeaderLen = len(magicChunk) + checksumSize + chunkHeaderSize
81 | obufLen = obufHeaderLen + maxEncodedLenOfMaxBlockSize
82 | )
83 |
84 | const (
85 | chunkTypeCompressedData = 0x00
86 | chunkTypeUncompressedData = 0x01
87 | chunkTypePadding = 0xfe
88 | chunkTypeStreamIdentifier = 0xff
89 | )
90 |
91 | var crcTable = crc32.MakeTable(crc32.Castagnoli)
92 |
93 | // crc implements the checksum specified in section 3 of
94 | // https://github.com/google/snappy/blob/master/framing_format.txt
95 | func crc(b []byte) uint32 {
96 | c := crc32.Update(0, crcTable, b)
97 | return uint32(c>>15|c<<17) + 0xa282ead8
98 | }
99 |
--------------------------------------------------------------------------------
/handler_socket2/compress/snappy/testdata/Mark.Twain-Tom.Sawyer.txt.rawsnappy:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/HowRareIs/solproxy/aec513a3c74cc390704f0a78a26a23e8bd06f318/handler_socket2/compress/snappy/testdata/Mark.Twain-Tom.Sawyer.txt.rawsnappy
--------------------------------------------------------------------------------
/handler_socket2/compress/status.go:
--------------------------------------------------------------------------------
1 | package compress
2 |
3 | import (
4 | "fmt"
5 | "sync"
6 | "time"
7 |
8 | "github.com/slawomir-pryczek/HSServer/handler_socket2/hscommon"
9 | )
10 |
11 | const thread_stopped_symbol = 'x'
12 |
13 | type stat_items struct {
14 | compression_count int
15 | compression_pieces int
16 | data_in_size uint64
17 | data_out_size uint64
18 |
19 | buffer_too_small int
20 | compression_ratio_too_low int
21 | }
22 |
23 | type stat struct {
24 | name string
25 | s_totals stat_items
26 | s_last_70 [7]stat_items
27 | curr_slice int
28 |
29 | mu sync.Mutex
30 |
31 | thread_symbol []byte
32 | thread_running []byte
33 | thread_items_done []int
34 | }
35 |
36 | func MakeStat(name string) *stat {
37 |
38 | ret := &stat{}
39 | ret.name = name
40 | ret.thread_symbol = make([]byte, 0, 30)
41 | ret.thread_running = make([]byte, 0, 30)
42 | go func() {
43 |
44 | last_slice := -1
45 | for {
46 | time.Sleep(500 * time.Millisecond)
47 | slice := int(time.Now().Unix()/10) % 7
48 | if slice == last_slice {
49 | continue
50 | }
51 | last_slice = slice
52 |
53 | ret.mu.Lock()
54 | ret.curr_slice = slice
55 | ret.s_last_70[slice] = stat_items{}
56 | ret.mu.Unlock()
57 | }
58 | }()
59 |
60 | return ret
61 | }
62 |
63 | func (this *stat) ThreadAdd(symbol byte) int {
64 | this.mu.Lock()
65 | this.thread_symbol = append(this.thread_symbol, symbol)
66 | this.thread_running = append(this.thread_running, thread_stopped_symbol)
67 | this.thread_items_done = append(this.thread_items_done, 0)
68 | ret := len(this.thread_symbol) - 1
69 | this.mu.Unlock()
70 | return ret
71 | }
72 |
73 | func (this *stat) ThreadSetRunning(thread_id int, is_running bool) {
74 | this.mu.Lock()
75 | if is_running {
76 | this.thread_running[thread_id] = this.thread_symbol[thread_id]
77 | } else {
78 | this.thread_running[thread_id] = thread_stopped_symbol
79 | }
80 | this.mu.Unlock()
81 | }
82 |
83 | func (this *stat) doStats(thread_id, piece_no, data_in_size, data_out_size int) {
84 | this.mu.Lock()
85 | if piece_no == 0 {
86 | this.s_totals.compression_count++
87 | }
88 | this.s_totals.compression_pieces++
89 | this.s_totals.data_in_size += uint64(data_in_size)
90 | this.s_totals.data_out_size += uint64(data_out_size)
91 |
92 | _slice := this.curr_slice
93 | if piece_no == 0 {
94 | this.s_last_70[_slice].compression_count++
95 | }
96 | this.s_last_70[_slice].compression_pieces++
97 | this.s_last_70[_slice].data_in_size += uint64(data_in_size)
98 | this.s_last_70[_slice].data_out_size += uint64(data_out_size)
99 | this.thread_running[thread_id] = thread_stopped_symbol
100 | this.thread_items_done[thread_id]++
101 | this.mu.Unlock()
102 | }
103 |
104 | func (this *stat) reportBufferTooSmall() {
105 | this.mu.Lock()
106 | this.s_totals.buffer_too_small++
107 | this.s_last_70[this.curr_slice].buffer_too_small++
108 | this.mu.Unlock()
109 | }
110 |
111 | func (this *stat) reportCompressionRatioTooLow() {
112 | this.mu.Lock()
113 | this.s_totals.compression_ratio_too_low++
114 | this.s_last_70[this.curr_slice].compression_ratio_too_low++
115 | this.mu.Unlock()
116 | }
117 |
118 | func (this *stat) GetStatus() string {
119 |
120 | table := hscommon.NewTableGen("Time", "Compressions", "Pieces", "Data In", "Data Out", "Ratio", "E-RLow", "E-Buffer")
121 | table.SetClass("tab threads")
122 |
123 | this.mu.Lock()
124 |
125 | _num := func(data ...int) string {
126 | if len(data) == 1 {
127 | return fmt.Sprintf("%d", data[0])
128 | }
129 | zeros := fmt.Sprintf("%d", data[1])
130 | return fmt.Sprintf("%0"+zeros+"d", data[0])
131 | }
132 | _addrow := func(title string, i stat_items) {
133 | ratio := " - "
134 | if i.data_out_size > 0 {
135 | __r := (i.data_in_size * 100) / i.data_out_size
136 | ratio = fmt.Sprintf("x%.2f", float64(__r)/100)
137 | }
138 | table.AddRow(title, _num(i.compression_count), _num(i.compression_pieces),
139 | hscommon.FormatBytes(i.data_in_size), hscommon.FormatBytes(i.data_out_size), ratio,
140 | _num(i.compression_ratio_too_low), _num(i.buffer_too_small))
141 |
142 | }
143 |
144 | i_last_10 := stat_items{}
145 | i_last_60 := stat_items{}
146 | for i := 1; i < len(this.s_last_70); i++ {
147 | pos := (this.curr_slice + len(this.s_last_70) - i) % len(this.s_last_70)
148 |
149 | sr := this.s_last_70[pos]
150 | if i == 1 {
151 | i_last_10 = sr
152 | }
153 | i_last_60.buffer_too_small += sr.buffer_too_small
154 | i_last_60.compression_count += sr.compression_count
155 | i_last_60.compression_pieces += sr.compression_pieces
156 | i_last_60.compression_ratio_too_low += sr.compression_ratio_too_low
157 | i_last_60.data_in_size += sr.data_in_size
158 | i_last_60.data_out_size += sr.data_out_size
159 | }
160 | table.AddRow(this.name)
161 | _addrow("Last 10s", i_last_10)
162 | _addrow("Last 60s", i_last_60)
163 | _addrow("Total", this.s_totals)
164 |
165 | tab_threads := hscommon.NewTableGen("Thread", "Items Done", "_class")
166 | tab_threads.SetClass("tab compressing")
167 |
168 | _add_thread := func(thr_num int, thr_status byte, thr_type_filter byte) {
169 | if thr_type_filter != this.thread_symbol[thr_num] {
170 | return
171 | }
172 | class := ""
173 | status := "⚫ "
174 | if thr_status != 'x' {
175 | status = "▶ "
176 | class = "running"
177 | }
178 | status += _num(thr_num, 3)
179 | status += " (" + string(this.thread_symbol[thr_num]) + ")"
180 | tab_threads.AddRow(status, _num(this.thread_items_done[thr_num]), class)
181 | }
182 |
183 | for thr_num, status := range this.thread_running {
184 | _add_thread(thr_num, status, 'N')
185 | }
186 | for thr_num, status := range this.thread_running {
187 | _add_thread(thr_num, status, 'F')
188 | }
189 | this.mu.Unlock()
190 |
191 | ret := table.Render()
192 | ret += tab_threads.RenderHorizFlat(14)
193 | return ret
194 | }
195 |
--------------------------------------------------------------------------------
/handler_socket2/compression_ex.go:
--------------------------------------------------------------------------------
1 | package handler_socket2
2 |
3 | import (
4 | "fmt"
5 | "github.com/slawomir-pryczek/HSServer/handler_socket2/compress"
6 | "github.com/slawomir-pryczek/HSServer/handler_socket2/config"
7 | "runtime"
8 | "strings"
9 | )
10 |
11 | var compressor_snappy *compress.Compressor = nil
12 | var compressor_flate *compress.Compressor = nil
13 |
14 | func compression_ex_read_config() {
15 | compression_support := config.Config().Get("COMPRESSION", "mp-flate")
16 | if strings.Index(compression_support, "mp-flate") > -1 {
17 | compressor_flate = compress.CreateCompressor(runtime.NumCPU(), compress.MakeFlate())
18 | }
19 | if strings.Index(compression_support, "mp-snappy") > -1 {
20 | compressor_snappy = compress.CreateCompressor(runtime.NumCPU(), compress.MakeSnappy())
21 | }
22 |
23 | if compressor_flate == nil && compressor_snappy == nil {
24 | fmt.Println("Multipart compression is disabled, use compression_support=[mp-flate,mp-snappy] to enable")
25 | } else {
26 | fmt.Println("Multipart compression is enabled")
27 | }
28 | }
29 |
30 | func compression_ex_status() string {
31 | if compressor_flate == nil && compressor_snappy == nil {
32 | return ""
33 | }
34 |
35 | ret := ""
36 | ret += "-- Multipart Compress (multi threaded)\n"
37 | ret += "Multipart compression is used to quickly set compressed data using internal framing format\n"
38 | ret += "It is not compatible with standard compression schemas.\n\n"
39 | ret += "E-RLow - error, compression ratio too low\tE-Buffer - error, compression buffer too small\n"
40 | if compressor_flate != nil {
41 | ret += compressor_flate.GetStatus()
42 | }
43 | if compressor_snappy != nil {
44 | ret += compressor_snappy.GetStatus()
45 | }
46 | return ret
47 | }
48 |
--------------------------------------------------------------------------------
/handler_socket2/config/config.go:
--------------------------------------------------------------------------------
1 | package config
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "strconv"
7 | "strings"
8 | "sync"
9 | "sync/atomic"
10 | "time"
11 | )
12 |
13 | var _config atomic.Value
14 |
15 | var cfg_initialized = false
16 | var cfg_mu sync.Mutex
17 | var cfg_onchange []func()
18 |
19 | func attachOnChange(callback func()) {
20 | cfg_mu.Lock()
21 | cfg_onchange = append(cfg_onchange, callback)
22 | _i := cfg_initialized
23 | cfg_mu.Unlock()
24 |
25 | if _i {
26 | callback()
27 | }
28 | }
29 |
30 | func init() {
31 | ReadConfig()
32 |
33 | go func() {
34 | for {
35 | time.Sleep(5 * time.Second)
36 |
37 | _cfg := Config()
38 | st, err := os.Stat(_cfg.cfg_file_path)
39 | if err != nil {
40 | continue
41 | }
42 |
43 | cfg_mu.Lock()
44 | _ci := cfg_initialized
45 | cfg_mu.Unlock()
46 | if !_ci {
47 | continue
48 | }
49 |
50 | file_changed := false
51 | file_changed = file_changed || _cfg.cfg_file_size != st.Size()
52 | file_changed = file_changed || _cfg.cfg_file_modified != st.ModTime().Unix()
53 | if !file_changed {
54 | continue
55 | }
56 |
57 | fmt.Println("\n\nConfig file changed, re-reading configuration!")
58 | tmp, err := _cfg_load_config()
59 | if err != nil {
60 | continue
61 | }
62 | tmp._cfg_local_interfaces()
63 | tmp._cfg_conditional_config()
64 | _config.Store(tmp)
65 |
66 | cbs := make([]func(), len(cfg_onchange))
67 | cfg_mu.Lock()
68 | copy(cbs, cfg_onchange)
69 | cfg_mu.Unlock()
70 | for _, cb := range cbs {
71 | go func() { cb() }()
72 | }
73 | }
74 | }()
75 | }
76 |
77 | func ReadConfig() {
78 |
79 | cfg_mu.Lock()
80 | defer cfg_mu.Unlock()
81 | if cfg_initialized {
82 | return
83 | }
84 | cfg_initialized = true
85 |
86 | tmp, err := _cfg_load_config()
87 | if err != nil {
88 | fmt.Println("===")
89 | fmt.Println("FATAL Error opening configuration file:\n", err.Error())
90 | fmt.Println("===")
91 | os.Exit(1)
92 | }
93 |
94 | tmp._cfg_local_interfaces()
95 | tmp._cfg_conditional_config()
96 | _config.Store(tmp)
97 | }
98 |
99 | func Config() *cfg {
100 | ret := _config.Load()
101 | if ret != nil {
102 | return ret.(*cfg)
103 | }
104 |
105 | ReadConfig()
106 | return _config.Load().(*cfg)
107 | }
108 |
109 | func (this *cfg) GetIPDistance(remote_addr string) byte {
110 |
111 | is_local := false
112 | for _, ip := range this.local_interfaces {
113 | if strings.Compare(ip, remote_addr) == 0 {
114 | is_local = true
115 | break
116 | }
117 | }
118 |
119 | if is_local {
120 | return 0
121 | }
122 | return 1
123 | }
124 |
125 | func (this *cfg) GetRawData(attr string, def string) interface{} {
126 | if val, ok := this.raw_data[attr]; ok {
127 | return val
128 | }
129 | return def
130 | }
131 |
132 | func (this *cfg) Get(attr, def string) string {
133 | if val, ok := this.config[attr]; ok {
134 | return val
135 | }
136 | return def
137 | }
138 |
139 | func (this *cfg) GetB(attr string) bool {
140 |
141 | if val, ok := this.config[attr]; ok && val == "1" {
142 | return true
143 | }
144 | return false
145 | }
146 |
147 | func (this *cfg) GetI(attr string, def int) int {
148 |
149 | if _, ok := this.config[attr]; !ok {
150 | return def
151 | }
152 |
153 | if ret, err := strconv.ParseInt(this.config[attr], 10, 64); err == nil {
154 | return int(ret)
155 | }
156 | return def
157 | }
158 |
159 | func (this *cfg) GetCompressionThreshold() int {
160 | return this.compression_threshold
161 | }
162 |
--------------------------------------------------------------------------------
/handler_socket2/config/subattr.go:
--------------------------------------------------------------------------------
1 | package config
2 |
3 | import (
4 | "encoding/json"
5 | "errors"
6 | "fmt"
7 | "strconv"
8 | "strings"
9 | )
10 |
11 | func (this *cfg) ValidateAttribs(attr string, subattrs []string) (bool, error) {
12 | if len(subattrs) == 0 {
13 | if _, exists := this.config[attr]; exists {
14 | return true, nil
15 | }
16 | return false, nil
17 | }
18 |
19 | has_any := false
20 | has_all := true
21 | missing := make([]string, 0, len(subattrs))
22 | for _, subattr := range subattrs {
23 | _, err := this._get_subattr_interface(attr, subattr)
24 | has_any = has_any || err == nil
25 | if err != nil {
26 | has_all = false
27 | missing = append(missing, subattr)
28 | }
29 | }
30 |
31 | if !has_any {
32 | return false, nil
33 | }
34 | if has_all {
35 | return true, nil
36 | }
37 |
38 | return false, errors.New("Attributes missing: " + strings.Join(missing, ", "))
39 | }
40 |
41 | func (this *cfg) _get_subattr_interface(attr string, subattr string) (interface{}, error) {
42 | i := (interface{})(nil)
43 | if val, exists := this.raw_data[attr]; exists {
44 | tmp := val.(map[string]interface{})
45 | if val, exists := tmp[subattr]; exists {
46 | i = val
47 | }
48 | }
49 | if i == nil {
50 | return 0, errors.New(fmt.Sprintf("Attribute %s/%s not found", attr, subattr))
51 | }
52 | return i, nil
53 | }
54 |
55 | func (this *cfg) GetSubattrInt(attr string, subattr string) (int, error) {
56 | i, err := this._get_subattr_interface(attr, subattr)
57 | if err != nil {
58 | return 0, err
59 | }
60 |
61 | switch i.(type) {
62 | case string:
63 | out, err := strconv.Atoi(i.(string))
64 | if err != nil {
65 | return 0, errors.New(fmt.Sprintf("Attribute %s/%s error:\n"+err.Error(), attr, subattr))
66 | }
67 | return out, nil
68 | case int:
69 | return i.(int), nil
70 | case float64:
71 | return int(i.(float64)), nil
72 | case json.Number:
73 | out, err := i.(json.Number).Int64()
74 | if err != nil {
75 | return 0, errors.New(fmt.Sprintf("Attribute %s/%s error:\n"+err.Error(), attr, subattr))
76 | }
77 | return int(out), nil
78 | case bool:
79 | if i.(bool) {
80 | return 1, nil
81 | } else {
82 | return 0, nil
83 | }
84 | default:
85 | return 0, errors.New(fmt.Sprintf("Attribute %s/%s is of wrong type (int required)", attr, subattr))
86 | }
87 | }
88 |
89 | func (this *cfg) GetSubattrString(attr string, subattr string) (string, error) {
90 | i, err := this._get_subattr_interface(attr, subattr)
91 | if err != nil {
92 | return "", err
93 | }
94 |
95 | switch i.(type) {
96 | case string:
97 | return i.(string), nil
98 | case int:
99 | return strconv.Itoa(i.(int)), nil
100 | case float64:
101 | return strconv.FormatFloat(i.(float64), 'f', 3, 64), nil
102 | case json.Number:
103 | return i.(json.Number).String(), nil
104 | case bool:
105 | if i.(bool) {
106 | return "1", nil
107 | } else {
108 | return "0", nil
109 | }
110 | default:
111 | return "", errors.New(fmt.Sprintf("Attribute %s/%s is of wrong type (int required)", attr, subattr))
112 | }
113 | }
114 |
--------------------------------------------------------------------------------
/handler_socket2/conn_ex.go:
--------------------------------------------------------------------------------
1 | package handler_socket2
2 |
3 | import (
4 | "fmt"
5 | "net"
6 | "strings"
7 |
8 | "github.com/slawomir-pryczek/HSServer/handler_socket2/compress"
9 | "github.com/slawomir-pryczek/HSServer/handler_socket2/config"
10 | )
11 |
12 | type conninfo struct {
13 | conn *net.TCPConn
14 | remote_distance byte
15 |
16 | comp *compress.Compressor
17 | compression_threshold int
18 | }
19 |
20 | func make_conn_ex(conn *net.TCPConn) conninfo {
21 | remote_addr := strings.Split(conn.RemoteAddr().String(), ":")[0]
22 | remote_distance := config.Config().GetIPDistance(remote_addr)
23 |
24 | conn_ex := conninfo{}
25 | conn_ex.conn = conn
26 | conn_ex.remote_distance = remote_distance
27 | if remote_distance > 0 {
28 | if compressor_flate != nil {
29 | conn_ex.comp = compressor_flate
30 | }
31 | conn_ex.compression_threshold = config.Config().GetI("compression_threshold", config.DEFAULT_COMPRESSION_THRESHOLD)
32 | }
33 |
34 | if conn_ex.compression_threshold == 0 {
35 | conn_ex.comp = nil
36 | }
37 | return conn_ex
38 | }
39 |
40 | func handle_conn_ex(data *HSParams, conn_ex *conninfo) {
41 | if conn_ex.remote_distance == 0 {
42 | return
43 | }
44 |
45 | features := strings.ToLower(data.GetParam("features", ""))
46 | has_snappy := strings.Contains(features, "snappy")
47 |
48 | // always use no compression for LOCAL(0), snappy or flate for LAN(1) and flate for WAN(2)
49 | if has_snappy && compressor_snappy != nil {
50 | conn_ex.comp = compressor_snappy
51 | }
52 | if (conn_ex.comp == nil || conn_ex.remote_distance > 1) && compressor_flate != nil {
53 | conn_ex.comp = compressor_flate
54 | }
55 |
56 | // update threshold
57 | _thr_from_cfg := config.Config().GetCompressionThreshold()
58 | conn_ex.compression_threshold = data.GetParamI("compression_threshold", _thr_from_cfg)
59 | if conn_ex.compression_threshold == 0 {
60 | conn_ex.comp = nil
61 | }
62 |
63 | _algo := "-"
64 | if conn_ex.comp != nil {
65 | _algo = conn_ex.comp.GetID()
66 | }
67 |
68 | fmt.Println("\t Conn-Ex <- ", conn_ex.conn.RemoteAddr(),
69 | " Network distance:", conn_ex.remote_distance,
70 | " size >", conn_ex.compression_threshold,
71 | " = ", _algo)
72 | }
73 |
--------------------------------------------------------------------------------
/handler_socket2/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/slawomir-pryczek/HSServer/handler_socket2
2 |
3 | go 1.18
4 |
5 | require github.com/golang/snappy v0.0.4
6 |
--------------------------------------------------------------------------------
/handler_socket2/go.sum:
--------------------------------------------------------------------------------
1 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
2 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
3 |
--------------------------------------------------------------------------------
/handler_socket2/handle_echo/echo.go:
--------------------------------------------------------------------------------
1 | package handle_echo
2 |
3 | import (
4 | "encoding/json"
5 | "strings"
6 |
7 | "github.com/slawomir-pryczek/HSServer/handler_socket2"
8 | )
9 |
10 | type HandleEcho struct {
11 | }
12 |
13 | func (this *HandleEcho) Initialize() {
14 | handler_socket2.StatusPluginRegister(func() (string, string) {
15 | return "Echo", "Echo plugin is enabled"
16 | })
17 | }
18 |
19 | func (this *HandleEcho) Info() string {
20 | return "This plugin will send back received data"
21 | }
22 |
23 | func (this *HandleEcho) GetActions() []string {
24 | return []string{"echo"}
25 | }
26 |
27 | func (this *HandleEcho) HandleAction(action string, data *handler_socket2.HSParams) string {
28 |
29 | ret := map[string]string{}
30 |
31 | in := data.GetParam("data", "")
32 | repeat := data.GetParamI("repeat", 1)
33 | if repeat < 1 {
34 | repeat = 1
35 | ret["warning"] = "Repeat must be at least 1"
36 | }
37 | if repeat > 500 {
38 | repeat = 500
39 | ret["warning"] = "Repeat must be at most 500"
40 | }
41 |
42 | if repeat > 1 {
43 | ret["data"] = strings.Repeat(in, repeat)
44 | } else {
45 | ret["data"] = in
46 | }
47 |
48 | _tmp, _ := json.Marshal(ret)
49 | data.FastReturnBNocopy(_tmp)
50 | return ""
51 | }
52 |
--------------------------------------------------------------------------------
/handler_socket2/handle_profiler/profiler.go:
--------------------------------------------------------------------------------
1 | package handle_profiler
2 |
3 | import (
4 | "fmt"
5 | "runtime"
6 |
7 | "github.com/slawomir-pryczek/HSServer/handler_socket2"
8 | "github.com/slawomir-pryczek/HSServer/handler_socket2/hscommon"
9 | )
10 |
11 | type HandleProfiler struct {
12 | }
13 |
14 | func (this *HandleProfiler) Initialize() {
15 |
16 | handler_socket2.StatusPluginRegister(func() (string, string) {
17 |
18 | tmp := this.HandleAction("profiler", &handler_socket2.HSParams{})
19 | return "Memory profile", tmp
20 |
21 | })
22 |
23 | }
24 |
25 | func (this *HandleProfiler) Info() string {
26 | return "This plugin will display basic memory profiling information"
27 | }
28 |
29 | func (this *HandleProfiler) GetActions() []string {
30 | return []string{"profiler"}
31 | }
32 |
33 | func (this *HandleProfiler) HandleAction(action string, data *handler_socket2.HSParams) string {
34 |
35 | memStats := &runtime.MemStats{}
36 | runtime.ReadMemStats(memStats)
37 |
38 | if data.GetParam("simple", "") == "1" {
39 |
40 | ret := ""
41 | ret += "memStats.TotalAlloc: " + hscommon.FormatBytes(memStats.TotalAlloc) + "\n"
42 | ret += "memStats.Mallocs: " + fmt.Sprintf("%d", memStats.Mallocs) + "\n"
43 | ret += "memStats.NumGC: " + fmt.Sprintf("%d", memStats.NumGC) + "\n"
44 | return ret
45 |
46 | }
47 |
48 | tg := hscommon.NewTableGen("Parameter", "Value", "Help")
49 | tg.SetClass("tab")
50 |
51 | tg.AddRow("General statistics ==== ")
52 | tg.AddRow("memStats.Alloc", hscommon.FormatBytes(memStats.Alloc), "Bytes allocated and still in use")
53 | tg.AddRow("memStats.TotalAlloc", hscommon.FormatBytes(memStats.TotalAlloc), "Bytes allocated in total (even if freed)")
54 | tg.AddRow("memStats.Sys", hscommon.FormatBytes(memStats.Sys), "Bytes obtained from system")
55 | tg.AddRow("memStats.Lookups", fmt.Sprintf("%d", memStats.Lookups), "Number of pointer lookups")
56 | tg.AddRow("memStats.Mallocs", fmt.Sprintf("%d", memStats.Mallocs), "Number of mallocs")
57 | tg.AddRow("memStats.Frees", fmt.Sprintf("%d", memStats.Frees), "Number of frees")
58 |
59 | tg.AddRow("Heap statistics ==== ")
60 | tg.AddRow("memStats.HeapAlloc", hscommon.FormatBytes(memStats.HeapAlloc), "Bytes allocated and still in use")
61 | tg.AddRow("memStats.HeapSys", hscommon.FormatBytes(memStats.HeapSys), "Bytes obtained from system")
62 | tg.AddRow("memStats.HeapIdle", hscommon.FormatBytes(memStats.HeapIdle), "Bytes in idle spans")
63 | tg.AddRow("memStats.HeapInuse", hscommon.FormatBytes(memStats.HeapInuse), "Bytes in non-idle span")
64 | tg.AddRow("memStats.HeapReleased", hscommon.FormatBytes(memStats.HeapReleased), "Bytes released to the OS")
65 | tg.AddRow("memStats.HeapObjects", fmt.Sprintf("%d", memStats.HeapObjects), "Total number of allocated objects")
66 |
67 | tg.AddRow("Stack & System allocation ==== ")
68 | tg.AddRow("memStats.StackInuse", hscommon.FormatBytes(memStats.StackInuse), "Bytes used by stack allocator")
69 | tg.AddRow("memStats.StackSys", hscommon.FormatBytes(memStats.StackSys), "Bytes obtained from OS by stack allocator")
70 | tg.AddRow("memStats.MSpanInuse", fmt.Sprintf("%d", memStats.MSpanInuse), "MSpan structures in use")
71 | tg.AddRow("memStats.MSpanSys", fmt.Sprintf("%d", memStats.MSpanSys), "MSpan structures obtained from OS")
72 | tg.AddRow("memStats.MCacheInuse", fmt.Sprintf("%d", memStats.MCacheInuse), "MCache structures in use")
73 | tg.AddRow("memStats.MCacheSys", fmt.Sprintf("%d", memStats.MCacheSys), "MCache structures obtained from OS")
74 | tg.AddRow("memStats.BuckHashSys", fmt.Sprintf("%d", memStats.BuckHashSys), "Profiling bucket hash table")
75 | tg.AddRow("memStats.OtherSys", fmt.Sprintf("%d", memStats.OtherSys), "Other system allocations")
76 |
77 | tg.AddRow("Garbage Collection ==== ")
78 | tg.AddRow("memStats.NumGC", fmt.Sprintf("%d", memStats.NumGC), "Number of garbage colledtions")
79 | tg.AddRow("memStats.PauseTotalNs", fmt.Sprintf("%d", memStats.PauseTotalNs), "Total time waited in GC")
80 | tg.AddRow("memStats.NextGC", fmt.Sprintf("%d", memStats.NextGC), "Next collection will happen when HeapAlloc ≥ this amount")
81 | tg.AddRow("memStats.LastGC", fmt.Sprintf("%d", memStats.LastGC), "End time of last collection (nanoseconds since 1970)")
82 |
83 | tg.AddRow("Server statistics ==== ")
84 | tg.AddRow("runtime.NumGoroutine", fmt.Sprintf("%d", runtime.NumGoroutine()), "Number of goroutines that currently exist")
85 | //tg.AddRow("runtime.InUseBytes", fmt.Sprintf("%d", runtime.InUseBytes()), "Number of bytes in use")
86 | //tg.AddRow("runtime.InUseObjects", fmt.Sprintf("%d", runtime.InUseObjects()), "Number of objects in use")
87 |
88 | return tg.Render()
89 | }
90 |
--------------------------------------------------------------------------------
/handler_socket2/hs_engine.go:
--------------------------------------------------------------------------------
1 | package handler_socket2
2 |
3 | import (
4 | "fmt"
5 | "github.com/slawomir-pryczek/HSServer/handler_socket2/config"
6 | "net"
7 | "os"
8 | "sync"
9 | )
10 |
11 | type ActionHandler interface {
12 | Initialize()
13 | Info() string
14 | GetActions() []string
15 | HandleAction(action string, data *HSParams) string
16 | }
17 |
18 | var action_handlers = make([]ActionHandler, 0)
19 | var actionToHandlerNum = make(map[string]int)
20 |
21 | func RegisterHandler(handlers ...ActionHandler) {
22 |
23 | for _, handler := range handlers {
24 | handler.Initialize()
25 | action_handlers = append(action_handlers, handler)
26 | }
27 |
28 | for hindex, handler := range action_handlers {
29 | for _, action := range handler.GetActions() {
30 | actionToHandlerNum[action] = hindex
31 | }
32 | }
33 | }
34 |
35 | type StatusPlugin func() (string, string)
36 |
37 | var statusPlugins = make([]StatusPlugin, 0)
38 |
39 | func StatusPluginRegister(f StatusPlugin) {
40 | statusPlugins = append(statusPlugins, f)
41 | }
42 |
43 | var boundTo []string = []string{}
44 | var boundMutex sync.Mutex
45 |
46 | func StartServer(bind_to []string) {
47 |
48 | compression_ex_read_config()
49 |
50 | var wg sync.WaitGroup
51 | wg.Add(1)
52 |
53 | for _, bt := range bind_to {
54 | if len(bt) < 1 {
55 | continue
56 | }
57 |
58 | go func(bt string) {
59 | switch {
60 | case bt[0] == 'h':
61 | startServiceHTTP(bt[1:], handleRequest)
62 |
63 | case bt[0] == 'u':
64 | startServiceUDP(bt[1:], handleRequest)
65 |
66 | default:
67 | startService(bt, handleRequest)
68 | }
69 |
70 | if config.Config().Get("FORCE_START", "") == "1" {
71 | fmt.Println("WARNING: Can't bind to all interfaces, but FORCE_START in effect")
72 | } else {
73 | fmt.Fprintf(os.Stderr, "Cannot bind to: %s or unexpected thread exit\n", bt)
74 | os.Exit(1)
75 | }
76 |
77 | }(bt)
78 | }
79 |
80 | wg.Wait()
81 | }
82 |
83 | type handlerFunc func(*HSParams) string
84 |
85 | // this is socket handler, it'll just start the socket and then pass flow to serveSocket
86 | // that'll act as socket driver
87 | func startService(bindTo string, handler handlerFunc) {
88 |
89 | tcpAddr, err := net.ResolveTCPAddr("tcp4", bindTo)
90 |
91 | if err != nil {
92 | fmt.Printf("Error resolving address: %s, %s\n", bindTo, err)
93 | return
94 | }
95 |
96 | listener, err := net.ListenTCP("tcp", tcpAddr)
97 | if err != nil {
98 | fmt.Printf("Error listening on TCP address: %s, %s\n", bindTo, err)
99 | return
100 | }
101 |
102 | fmt.Printf("Socket Service started : %s\n", bindTo)
103 | boundMutex.Lock()
104 | boundTo = append(boundTo, "socket:"+bindTo)
105 | boundMutex.Unlock()
106 |
107 | for {
108 | conn, err := listener.AcceptTCP()
109 | if err != nil {
110 | continue
111 | }
112 |
113 | go serveSocket(conn, handler)
114 | }
115 | }
116 |
117 | // limited, self-contained HTTP handler, with limited statistics and no
118 | // compression support
119 | type httpRequest struct {
120 | req string
121 | start_time int64
122 | end_time int64
123 | status string
124 | }
125 |
126 | var httpRequestStatus = make(map[uint64]*httpRequest)
127 | var httpRequestId uint64
128 | var httpStatMutex sync.Mutex
129 |
130 | func handleRequest(data *HSParams) string {
131 |
132 | action := data.GetParam("action", "")
133 |
134 | if action == "server-status" {
135 | return handlerServerStatus(data)
136 | }
137 |
138 | if config.CfgIsDebug() {
139 | fmt.Printf("Action %s\n", action)
140 | }
141 |
142 | if action == "" {
143 | return "Please specify action (0x1), or ?action=server-status for help"
144 | }
145 |
146 | if hindex, ok := actionToHandlerNum[action]; ok {
147 | return (action_handlers[hindex]).HandleAction(action, data)
148 | }
149 |
150 | return "Please specify action (0x2)"
151 | }
152 |
--------------------------------------------------------------------------------
/handler_socket2/hs_http.go:
--------------------------------------------------------------------------------
1 | package handler_socket2
2 |
3 | import (
4 | "fmt"
5 | "io/ioutil"
6 | "net/http"
7 | "sort"
8 | "strconv"
9 | "strings"
10 | "time"
11 |
12 | "github.com/slawomir-pryczek/HSServer/handler_socket2/hscommon"
13 | )
14 |
15 | type HTTPPlugin func(http.ResponseWriter, http.Header, map[string]string, []byte) bool
16 |
17 | var HTTPPlugins = make([]HTTPPlugin, 0)
18 |
19 | func HTTPPluginRegister(f HTTPPlugin) {
20 | HTTPPlugins = append(HTTPPlugins, f)
21 | }
22 |
23 | func startServiceHTTP(bindTo string, handler handlerFunc) {
24 |
25 | fmt.Printf("HTTP Service starting : %s\n", bindTo)
26 | boundMutex.Lock()
27 | boundTo = append(boundTo, "http://"+bindTo)
28 | boundMutex.Unlock()
29 |
30 | handle_hunc := func(w http.ResponseWriter, r *http.Request) {
31 | req_len := 0
32 |
33 | params := make(map[string]string)
34 | for k, v := range r.URL.Query() {
35 | v_ := strings.Join(v, ",")
36 | params[k] = v_
37 | req_len += len(k) + len(v)
38 | }
39 |
40 | for k, v := range r.Header {
41 | req_len += len(k) + len(v)
42 | }
43 |
44 | // build request representation from GET and POST
45 | str_req_id := r.URL.RawQuery
46 | r_body := make([]byte, 0)
47 | if r.Method == "POST" {
48 | tmp, err := ioutil.ReadAll(r.Body)
49 | if err == nil {
50 | r.Body.Close()
51 | r_body = tmp
52 |
53 | if len(r_body) > 40 {
54 | str_req_id += " P:" + string(r_body[0:40]) + "..."
55 | } else {
56 | str_req_id += " P:" + string(r_body)
57 | }
58 | }
59 | }
60 |
61 | _req_status := &httpRequest{str_req_id, time.Now().UnixNano(), 0, "R"}
62 | httpStatMutex.Lock()
63 | httpRequestId++
64 | _my_reqid := httpRequestId
65 | httpRequestStatus[_my_reqid] = _req_status
66 | httpStatMutex.Unlock()
67 |
68 | // plugins support, now we can process raw HTTP request
69 | for _, plugin := range HTTPPlugins {
70 | if plugin(w, r.Header, params, r_body) {
71 |
72 | _end := time.Now().UnixNano()
73 | go func(_my_reqid uint64, _end int64) {
74 | httpStatMutex.Lock()
75 | _req_status.status = "F"
76 | _req_status.end_time = _end
77 | httpStatMutex.Unlock()
78 |
79 | time.Sleep(5000 * time.Millisecond)
80 | httpStatMutex.Lock()
81 | delete(httpRequestStatus, _my_reqid)
82 | httpStatMutex.Unlock()
83 | }(_my_reqid, _end)
84 |
85 | return
86 | }
87 | }
88 |
89 | hsparams := CreateHSParamsFromMap(params)
90 |
91 | ret := []byte(handleRequest(hsparams))
92 | if hsparams.fastreturn != nil {
93 | ret = hsparams.fastreturn
94 | }
95 |
96 | w.Header().Add("Server", version)
97 | w.Header().Add("Content-type", "text/html")
98 | w.Header().Add("Content-length", strconv.Itoa(len(ret)))
99 |
100 | w.Header().Add("Cache-Control", "no-cache, no-store, must-revalidate")
101 | w.Header().Add("Pragma", "no-cache")
102 | w.Header().Add("Expires", "0")
103 | for _, v := range hsparams.additional_resp_headers {
104 | _pos := strings.IndexByte(v, ':')
105 | if _pos < 0 {
106 | continue
107 | }
108 | w.Header().Add(v[0:_pos], v[_pos+1:])
109 | }
110 |
111 | httpStatMutex.Lock()
112 | _req_status.status = "W"
113 | httpStatMutex.Unlock()
114 |
115 | w.Write(ret)
116 |
117 | httpStatMutex.Lock()
118 | _req_status.status = "F"
119 | _req_status.end_time = time.Now().UnixNano()
120 | httpStatMutex.Unlock()
121 |
122 | hsparams.Cleanup()
123 |
124 | go func(_my_reqid uint64) {
125 | time.Sleep(5000 * time.Millisecond)
126 | httpStatMutex.Lock()
127 | delete(httpRequestStatus, _my_reqid)
128 | httpStatMutex.Unlock()
129 | }(_my_reqid)
130 |
131 | }
132 |
133 | err := http.ListenAndServe(bindTo, http.HandlerFunc(handle_hunc))
134 | if err != nil {
135 | fmt.Println("HTTP Error listening on TCP address: ", bindTo)
136 | }
137 |
138 | }
139 |
140 | func GetStatusHTTP() string {
141 | httpStatMutex.Lock()
142 | now := time.Now().UnixNano()
143 | scored_items := make([]hscommon.ScoredItems, len(httpRequestStatus))
144 | i := 0
145 | for k, rs := range httpRequestStatus {
146 |
147 | took_str := "??"
148 | took := float64(0)
149 | if rs.start_time != 0 {
150 | if rs.end_time == 0 {
151 | took = float64(now-rs.start_time) / float64(1000000)
152 | } else {
153 | took = float64(rs.end_time-rs.start_time) / float64(1000000)
154 | }
155 | took_str = fmt.Sprintf("%.3fms", took)
156 | }
157 |
158 | tmp := fmt.Sprintf("Num. %d - [%s] %s %s
", k, rs.status, took_str, rs.req)
159 |
160 | scored_items[i] = hscommon.ScoredItems{Item: tmp, Score: int64(k)}
161 | i++
162 | }
163 |
164 | httpStatMutex.Unlock()
165 |
166 | sort.Sort(hscommon.SIArr(scored_items))
167 | tmp := ""
168 | for _, v := range scored_items {
169 | tmp += v.Item
170 | }
171 |
172 | return tmp
173 | }
174 |
--------------------------------------------------------------------------------
/handler_socket2/hs_udp_v1.go:
--------------------------------------------------------------------------------
1 | package handler_socket2
2 |
3 | import (
4 | "bytes"
5 | "compress/flate"
6 | "fmt"
7 | "github.com/slawomir-pryczek/HSServer/handler_socket2/config"
8 | "net/url"
9 | "strings"
10 | )
11 |
12 | func runRequest(key string, message_body []byte, is_compressed bool, handler handlerFunc) {
13 | // compression support!
14 |
15 | if config.CfgIsDebug() {
16 | fmt.Println("FROM UDP: ", string(message_body))
17 | }
18 |
19 | curr_msg := ""
20 | if is_compressed {
21 |
22 | r := flate.NewReader(bytes.NewReader(message_body))
23 |
24 | buf := new(bytes.Buffer)
25 | buf.ReadFrom(r)
26 | curr_msg = buf.String()
27 | r.Close()
28 |
29 | } else {
30 | curr_msg = string(message_body)
31 | }
32 | // <<
33 |
34 | // connection debugging
35 | _req_txt := ""
36 | _conn_data, _ := url.QueryUnescape(curr_msg)
37 | if len(_conn_data) > 60 {
38 |
39 | _pos := 0
40 | _conn_data_wbr := ""
41 | for _pos < len(_conn_data) {
42 | _end := _pos + 80
43 | if _end > len(_conn_data) {
44 | _end = len(_conn_data)
45 | }
46 | _conn_data_wbr += _conn_data[_pos:_end] + ""
47 | _pos += 80
48 | }
49 |
50 | _req_txt = "[...] " + _conn_data[0:60] + "" + _conn_data_wbr + "
"
51 | } else {
52 | _req_txt = "" + _conn_data + " "
53 | }
54 | // <<
55 |
56 | // process packet - parameters
57 | params, _ := url.ParseQuery(curr_msg)
58 | fmt.Print(params)
59 |
60 | params2 := make(map[string]string)
61 | for _k, _v := range params {
62 | params2[_k] = implode(",", _v)
63 | }
64 | // <<
65 |
66 | udpStatRequest(key, params2["action"], _req_txt)
67 |
68 | data := ""
69 | action := ""
70 | action_specified := false
71 | for {
72 |
73 | action, action_specified = params2["action"]
74 | if !action_specified || action == "" {
75 | action = "default"
76 | }
77 |
78 | hsparams := CreateHSParamsFromMap(params2)
79 | data = handler(hsparams)
80 | if hsparams.fastreturn != nil {
81 | data = string(hsparams.fastreturn)
82 | }
83 |
84 | //fmt.Println(data)
85 | if !strings.HasPrefix(data, "X-Forward:") {
86 |
87 | hsparams.Cleanup()
88 | break
89 | }
90 |
91 | var redir_vals url.Values
92 | var err error
93 | if redir_vals, err = url.ParseQuery(data[10:]); err != nil {
94 |
95 | data = "X-Forward, wrong redirect" + data
96 |
97 | hsparams.Cleanup()
98 | break
99 | }
100 |
101 | params2 = make(map[string]string)
102 | for rvk, _ := range redir_vals {
103 | params2[strings.TrimLeft(rvk, "?")] = redir_vals.Get(rvk)
104 | }
105 |
106 | hsparams.Cleanup()
107 | fmt.Println(params2)
108 | }
109 | }
110 |
111 | // return number of bytes we need to move forward
112 | // < -1 error - flush the buffer
113 | // 0 - needs more data to process the request
114 | // > 1 message processed correctly - move forward
115 | func processRequestData(key string, message []byte, handler handlerFunc) (int, []byte, bool) {
116 |
117 | var terminator = []byte("\r\n\r\n")
118 | const terminator_len = len("\r\n\r\n")
119 |
120 | lenf := bytes.Index(message, terminator)
121 | if lenf == -1 {
122 | return 0, nil, false
123 | }
124 |
125 | // request is terminated - we can start to process it!
126 | bytes_rec_uncompressed := 0
127 | is_compressed, required_size, guid := processHeader(message[0:lenf])
128 | fmt.Println("GUID: ", guid)
129 |
130 | if required_size < 0 {
131 | return -1, nil, false
132 | }
133 |
134 | // we also need to account for length header
135 | required_size += lenf + terminator_len
136 |
137 | // still need more data to arrive?
138 | if len(message) < required_size {
139 | return 0, nil, false
140 | }
141 |
142 | bytes_rec_uncompressed++
143 | return required_size, message[lenf+terminator_len : required_size], is_compressed
144 | }
145 |
--------------------------------------------------------------------------------
/handler_socket2/hscommon/str.go:
--------------------------------------------------------------------------------
1 | package hscommon
2 |
3 | import (
4 | "strings"
5 | )
6 |
7 | func _prefix_gen(s string, to_len int, with string) string {
8 |
9 | if len(s) >= to_len {
10 | return ""
11 | }
12 |
13 | _r := to_len - len(s)
14 | if len(with) > 1 {
15 | _r = (_r / len(with)) + 1
16 | }
17 | return strings.Repeat(with, _r)[0:_r]
18 | }
19 |
20 | func StrPrefix(s string, to_len int, with string) string {
21 | return _prefix_gen(s, to_len, with) + s
22 | }
23 |
24 | func StrPostfix(s string, to_len int, with string) string {
25 | return s + _prefix_gen(s, to_len, with)
26 | }
27 |
28 | func StrPrefixHTML(s string, to_len int, with string) string {
29 | html_len := len(s) - StrRealLen(s)
30 | return _prefix_gen(s, to_len+html_len, with) + s
31 | }
32 |
33 | func StrPostfixHTML(s string, to_len int, with string) string {
34 | html_len := len(s) - StrRealLen(s)
35 | return s + _prefix_gen(s, to_len+html_len, with)
36 | }
37 |
38 | func StrRealLen(s string) int {
39 |
40 | if len(s) == 0 {
41 | return 0
42 | }
43 |
44 | htmlTagStart := '<'
45 | htmlTagEnd := '>'
46 |
47 | sr := []rune(s)
48 | count := 0
49 | should_count := true
50 | last_char := sr[0]
51 | for _, c := range []rune(sr) {
52 | if c == htmlTagStart {
53 | should_count = false
54 | }
55 | if c == htmlTagEnd {
56 | should_count = true
57 | last_char = c
58 | continue
59 | }
60 | if should_count && (c != ' ' || last_char != ' ') {
61 | count++
62 | }
63 |
64 | last_char = c
65 | }
66 | return count
67 | }
68 |
69 | func StripHTML(s string) string {
70 | ret := ""
71 | htmlTagStart := '<'
72 | htmlTagEnd := '>'
73 |
74 | sr := []rune(s)
75 | should_count := true
76 | for _, c := range []rune(sr) {
77 | if c == htmlTagStart {
78 | should_count = false
79 | }
80 | if c == htmlTagEnd {
81 | should_count = true
82 | continue
83 | }
84 | if should_count {
85 | ret += string(c)
86 | }
87 | }
88 |
89 | return ret
90 | }
91 |
92 | func StrMessage(m string, is_ok bool) string {
93 | if is_ok {
94 | return " ⬤ " + m + " "
95 | } else {
96 | return " ⮿ " + m + " "
97 | }
98 | }
99 |
100 | func StrFirstChars(s string, max_len int) string {
101 | if max_len == 0 {
102 | return ""
103 | }
104 | if len(s) <= max_len {
105 | return s
106 | }
107 | return s[0:max_len] + "..."
108 | }
109 |
110 | func StrLastChars(s string, max_len int) string {
111 | if max_len == 0 {
112 | return ""
113 | }
114 | if len(s) <= max_len {
115 | return s
116 | }
117 | return "..." + s[len(s)-max_len:]
118 | }
119 |
120 | func StrMidChars(s string, max_len int) string {
121 | if max_len == 0 {
122 | return ""
123 | }
124 | max_len = (max_len + 1) / 2
125 | if len(s) <= max_len*2 {
126 | return s
127 | }
128 | return s[0:max_len] + "..." + s[len(s)-max_len:]
129 | }
130 |
--------------------------------------------------------------------------------
/handler_socket2/oslimits/oslimits_freebsd.go:
--------------------------------------------------------------------------------
1 | package oslimits
2 |
3 | func SetOpenFilesLimit(num int) bool {
4 | return true
5 | }
6 |
--------------------------------------------------------------------------------
/handler_socket2/oslimits/oslimits_linux.go:
--------------------------------------------------------------------------------
1 | package oslimits
2 |
3 | import (
4 | "fmt"
5 | "os"
6 | "runtime"
7 | "strings"
8 | "syscall"
9 | )
10 |
11 | func SetOpenFilesLimit(num int) bool {
12 |
13 | is_linux := strings.Contains(strings.ToLower(runtime.GOOS), "linux")
14 |
15 | if !is_linux {
16 | return false
17 | }
18 |
19 | var rLimit syscall.Rlimit
20 | err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
21 | if err != nil {
22 | fmt.Fprintf(os.Stderr, "Error Getting Rlimit "+err.Error())
23 | return false
24 | }
25 |
26 | rLimit.Max = uint64(num)
27 | rLimit.Cur = uint64(num)
28 |
29 | err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit)
30 | if err != nil {
31 | fmt.Fprintf(os.Stderr, "Error Setting Rlimit "+err.Error())
32 | return false
33 | }
34 |
35 | err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
36 | if err != nil {
37 | fmt.Fprintf(os.Stderr, "Error Getting Rlimit "+err.Error())
38 | return false
39 | }
40 |
41 | if rLimit.Max != uint64(num) || rLimit.Cur != uint64(num) {
42 | _tmp := fmt.Sprintf("Error Setting Rlimit, requested %d, got %d/%d ", num, rLimit.Cur, rLimit.Max)
43 | fmt.Fprintf(os.Stderr, _tmp)
44 | }
45 |
46 | return true
47 | }
48 |
--------------------------------------------------------------------------------
/handler_socket2/oslimits/oslimits_windows.go:
--------------------------------------------------------------------------------
1 | package oslimits
2 |
3 | func SetOpenFilesLimit(num int) bool {
4 | return true
5 | }
6 |
--------------------------------------------------------------------------------
/handler_socket2/stats/stats.go:
--------------------------------------------------------------------------------
1 | package stats
2 |
3 | import (
4 | "fmt"
5 | "sync"
6 | "time"
7 |
8 | "github.com/slawomir-pryczek/HSServer/handler_socket2/hscommon"
9 | )
10 |
11 | // #############################################################################
12 | // general stats
13 |
14 | var global_stats_connections uint64 = 0
15 | var global_stats_requests uint64 = 0
16 | var global_stats_errors uint64 = 0
17 | var global_stats_req_time uint64 = 0
18 | var global_stats_req_time_full uint64 = 0 // time of whole request (including received + sent)
19 |
20 | // #############################################################################
21 | // statistical structutes
22 |
23 | type Connection struct {
24 | id uint64
25 | ip string
26 | status string
27 | action string
28 | data string
29 | comment string
30 | is_error bool
31 | start_time int64
32 | end_time int64
33 |
34 | request_size, request_size_compressed, response_size uint64
35 | }
36 |
37 | type stats struct {
38 | requests uint64
39 | req_time uint64
40 | req_time_full uint64
41 |
42 | b_request_size uint64
43 | b_request_compressed uint64
44 |
45 | b_resp_size uint64
46 | b_resp_compressed uint64
47 |
48 | resp_skipped uint64
49 | resp_b_skipped uint64
50 | }
51 |
52 | func (this *stats) getAdjStats() stats {
53 |
54 | ret := stats{}
55 | ret.requests = this.requests
56 | ret.req_time = this.req_time
57 | ret.req_time_full = this.req_time_full
58 |
59 | ret.b_request_size = this.b_request_size
60 | ret.b_request_compressed = this.b_request_compressed
61 |
62 | ret.b_resp_size = this.b_resp_size - this.resp_b_skipped
63 | ret.b_resp_compressed = this.b_resp_compressed - this.resp_b_skipped
64 |
65 | ret.resp_skipped = this.resp_skipped
66 | ret.resp_b_skipped = this.resp_b_skipped
67 |
68 | return ret
69 | }
70 |
71 | var stats_mutex sync.Mutex
72 | var status_connections map[uint64]*Connection = make(map[uint64]*Connection)
73 | var stats_actions map[string]*stats = make(map[string]*stats)
74 |
75 | // #############################################################################
76 | // statistical structutes
77 |
78 | type Conn struct {
79 | conn_id uint64
80 | }
81 |
82 | const TYPE_NUMERIC_STAT = 0
83 | const TYPE_CONN_OPEN = 0
84 | const TYPE_CONN_CLOSE = 0
85 | const TYPE_CONN_UPDATE = 0
86 |
87 | type stat_event struct {
88 | e_type int
89 | conn_id uint64
90 |
91 | str1 string
92 | str2 string
93 | data [9]int
94 | }
95 |
96 | func MakeConnection(remoteAddr string) *Connection {
97 |
98 | now_us := time.Now().UnixNano()
99 |
100 | newconn := &Connection{}
101 | newconn.ip = remoteAddr
102 | newconn.status = "O"
103 | newconn.start_time = now_us
104 | newconn.end_time = 0
105 |
106 | stats_mutex.Lock()
107 | global_stats_connections++
108 | conn_id := global_stats_connections
109 | newconn.id = conn_id
110 | status_connections[conn_id] = newconn
111 | uh_add_unsafe(int(now_us/1000000000), 1, 0, 0, 0, 0, 0, 0, 0, 0, false)
112 | stats_mutex.Unlock()
113 |
114 | return newconn
115 | }
116 |
117 | func (this *Connection) StateReading() int64 {
118 |
119 | start := time.Now().UnixNano()
120 |
121 | stats_mutex.Lock()
122 | this.status = "Sr"
123 | this.start_time = start
124 | this.end_time = 0
125 | stats_mutex.Unlock()
126 |
127 | return start
128 |
129 | }
130 |
131 | func (this *Connection) StateServing(action, _pinfo string) {
132 |
133 | stats_mutex.Lock()
134 | this.status = "Ss1"
135 | this.action = action
136 | this.data = _pinfo
137 | stats_mutex.Unlock()
138 | }
139 |
140 | func (this *Connection) StateWriting(request_size, request_size_compressed, response_size uint64) uint64 {
141 |
142 | time_end := time.Now().UnixNano()
143 |
144 | stats_mutex.Lock()
145 | this.status = "W"
146 | this.end_time = time_end
147 |
148 | this.request_size = request_size
149 | this.request_size_compressed = request_size_compressed
150 | this.response_size = response_size
151 |
152 | took := uint64((time_end - this.start_time) / 1000)
153 | action := this.action
154 |
155 | if _, ok := stats_actions[action]; !ok {
156 | stats_actions[action] = &stats{}
157 | }
158 |
159 | _sa := stats_actions[action]
160 | _sa.requests++
161 | _sa.req_time += took
162 | _sa.b_request_size += request_size
163 | _sa.b_request_compressed += request_size_compressed
164 | _sa.b_resp_size += response_size
165 | stats_mutex.Unlock()
166 |
167 | return took
168 | }
169 |
170 | func (this *Connection) StateKeepalive(resp_compressed, took uint64, skip_response_sent bool) {
171 |
172 | now_us := time.Now().UnixNano()
173 |
174 | stats_mutex.Lock()
175 | _took_full := uint64((now_us - this.start_time) / 1000)
176 | global_stats_requests++
177 | global_stats_req_time += took
178 | global_stats_req_time_full += uint64(_took_full)
179 |
180 | action := this.action
181 | if _, ok := stats_actions[action]; !ok {
182 | stats_actions[action] = &stats{}
183 | }
184 | _sa := stats_actions[action]
185 |
186 | this.status = "K"
187 | _sa.b_resp_compressed += resp_compressed
188 | _sa.req_time_full += _took_full
189 | if skip_response_sent {
190 | _sa.resp_b_skipped += this.response_size
191 | _sa.resp_skipped++
192 | }
193 |
194 | uh_add_unsafe(int(now_us/1000000000), 0, 1, 0, took, _took_full, this.request_size, this.request_size_compressed, this.response_size, resp_compressed, skip_response_sent)
195 | stats_mutex.Unlock()
196 |
197 | }
198 |
199 | func (this *Connection) Close(comment string, is_error bool) {
200 |
201 | if is_error {
202 | stats_mutex.Lock()
203 | global_stats_errors++
204 | uh_add_unsafe(hscommon.TSNow(), 0, 0, 1, 0, 0, 0, 0, 0, 0, false)
205 | stats_mutex.Unlock()
206 | }
207 |
208 | x := func(conn_id uint64, comment string, is_error bool) {
209 |
210 | if is_error {
211 | fmt.Print("Error: " + comment)
212 | } else {
213 | fmt.Print("Info: " + comment)
214 | }
215 |
216 | time.Sleep(5000 * time.Millisecond)
217 | stats_mutex.Lock()
218 | var c *Connection = status_connections[conn_id]
219 | if c.end_time == 0 {
220 | c.start_time = 0
221 | }
222 | c.status = "X"
223 | c.comment = comment
224 | c.is_error = is_error
225 | stats_mutex.Unlock()
226 |
227 | time.Sleep(5000 * time.Millisecond)
228 | stats_mutex.Lock()
229 | delete(status_connections, conn_id)
230 | stats_mutex.Unlock()
231 | }
232 |
233 | go x(this.id, comment, is_error)
234 | }
235 |
--------------------------------------------------------------------------------
/handler_socket2/stats/stats_history.go:
--------------------------------------------------------------------------------
1 | package stats
2 |
3 | import (
4 | "github.com/slawomir-pryczek/HSServer/handler_socket2/hscommon"
5 | )
6 |
7 | type uh_struct struct {
8 | timestamp int
9 |
10 | connections uint64
11 | requests uint64
12 | errors uint64
13 | req_time uint64
14 | req_time_full uint64
15 |
16 | b_request_size uint64
17 | b_request_compressed uint64
18 |
19 | b_resp_size uint64
20 | b_resp_compressed uint64
21 |
22 | resp_skipped uint64
23 | resp_b_skipped uint64
24 | }
25 |
26 | var uptime_history [6]uh_struct
27 |
28 | func uh_add_unsafe(now int, connections, requests, errors, req_time, req_time_full, b_request_size, b_request_compressed, b_resp_size, b_resp_compressed uint64, skip_response_sent bool) {
29 |
30 | uh_now := &uptime_history[now%6]
31 | if uh_now.timestamp != now {
32 | uh_now.timestamp = now
33 | uh_now.connections = connections
34 | uh_now.requests = requests
35 | uh_now.errors = errors
36 | uh_now.req_time = req_time
37 | uh_now.req_time_full = req_time_full
38 |
39 | uh_now.b_request_size = b_request_size
40 | uh_now.b_request_compressed = b_request_compressed
41 | uh_now.b_resp_size = b_resp_size
42 | uh_now.b_resp_compressed = b_resp_compressed
43 |
44 | if skip_response_sent {
45 | uh_now.resp_skipped = requests
46 | uh_now.resp_b_skipped = b_resp_compressed
47 | } else {
48 | uh_now.resp_skipped, uh_now.resp_b_skipped = 0, 0
49 | }
50 |
51 | } else {
52 | uh_now.connections += connections
53 | uh_now.requests += requests
54 | uh_now.errors += errors
55 | uh_now.req_time += req_time
56 | uh_now.req_time_full += req_time_full
57 |
58 | uh_now.b_request_size += b_request_size
59 | uh_now.b_request_compressed += b_request_compressed
60 | uh_now.b_resp_size += b_resp_size
61 | uh_now.b_resp_compressed += b_resp_compressed
62 |
63 | if skip_response_sent {
64 | uh_now.resp_skipped += requests
65 | uh_now.resp_b_skipped += b_resp_compressed
66 | }
67 | }
68 |
69 | }
70 |
71 | func uh_get() uh_struct {
72 |
73 | ret := uh_struct{}
74 | now := hscommon.TSNow()
75 |
76 | stats_mutex.Lock()
77 | for i := 0; i < 6; i++ {
78 |
79 | // current second - we're still collecting stats
80 | if i == 0 {
81 | continue
82 | }
83 |
84 | now--
85 | uh_now := &uptime_history[now%6]
86 | if uh_now.timestamp != now {
87 | uh_now.timestamp = now
88 | uh_now.connections = 0
89 | uh_now.requests = 0
90 | uh_now.errors = 0
91 | uh_now.req_time = 0
92 | uh_now.req_time_full = 0
93 |
94 | uh_now.b_request_size = 0
95 | uh_now.b_request_compressed = 0
96 | uh_now.b_resp_size = 0
97 | uh_now.b_resp_compressed = 0
98 |
99 | uh_now.resp_skipped = 0
100 | uh_now.resp_b_skipped = 0
101 | continue
102 | }
103 |
104 | ret.connections += uh_now.connections
105 | ret.requests += uh_now.requests
106 | ret.errors += uh_now.errors
107 | ret.req_time += uh_now.req_time
108 | ret.req_time_full += uh_now.req_time_full
109 |
110 | ret.b_request_size += uh_now.b_request_size
111 | ret.b_request_compressed += uh_now.b_request_compressed
112 | ret.b_resp_size += uh_now.b_resp_size - uh_now.resp_b_skipped
113 | ret.b_resp_compressed += uh_now.b_resp_compressed - uh_now.resp_b_skipped
114 |
115 | ret.resp_skipped += uh_now.resp_skipped
116 | ret.resp_b_skipped += uh_now.resp_b_skipped
117 | }
118 | stats_mutex.Unlock()
119 | return ret
120 | }
121 |
--------------------------------------------------------------------------------
4 | 
31 | 
31 | 
14 |