(res) ?? new StatsRes();
113 | return ret.action;
114 | }
115 | public static void Logging(bool server, int fileid, string logs)
116 | {
117 | Console.WriteLine("\r\n" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss ") + logs);
118 | if (server)
119 | {
120 | try
121 | {
122 | HttpRequest("https://anyplots.com/api/v1/client/logging", new { token = ClientToken, id = fileid, data = logs });
123 | }
124 | catch (Exception ex)
125 | {
126 | try
127 | {
128 | HttpRequest("https://anyplots.com/api/v1/client/logging", new { token = ClientToken, id = fileid, data = logs });
129 | }
130 | catch { }
131 | }
132 | }
133 | }
134 | }
135 |
136 | }
137 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | cloud chia plotter: fast chia plotting without SSD disk or large amount of memory.
2 |
3 | It does not require SSD disk or large amount of memory, and can run well on low configuration rig, such as raspberry pi.
4 |
5 | https://anyplots.com/ The marketplace to buy and sell chia plots, starts at $0.20/plot.
6 |
7 | 
8 |
9 | Cloud Chia Plotter Usage
10 |
11 | Arguments:
12 |
13 | -p project token ( get it from your project page https://anyplots.com/buy-chia-plot/projects ), such as(40 chars):
14 | -p 000005e13f5fc456753081edf7dcc98986dcffa15
15 |
16 |
17 | -d directories for save plots, separate multiple directories with semicolons, such as:
18 | -d "/mnt/d/;/mnt/e/;/mnt/f/"
19 | -d "d:\;e:\;f:\"
20 |
21 | -b set your real network bandwidth/speed(Mbps), such as:
22 | for 500Mbps: -b 500 , for 1000Mbps: -b 1000
23 | please test your network speed at https://speedtest.net
24 |
25 | for example:
26 |
27 | .\\CloudChiaPlotter.exe -p 000005e13f5fc456753081edf7dcc98986dcffa15 -d "d:\\;e:\\;f:\\" -b 500
28 |
29 | ./CloudChiaPlotter -p 000005e13f5fc456753081edf7dcc98986dcffa15 -d "/mnt/d/;/mnt/e/;/mnt/f/" -b 500
30 |
31 | You can run multiple processes at the same time.
32 |
33 |
34 |
35 | Tips
36 | Do not set bandwidth parameter that are inconsistent with your actual bandwidth, Please test your network speed at
37 | https://speedtest.net
38 |
If you set -b 1000 for the actual bandwidth of 500mbps, it will reduce your download speed in most cases, because the probability of network packet loss will increase.
39 |
40 |
41 | For bandwidth greater than 500mbps, two processes can be run at the same time, but the bandwidth parameter should be divided by 2.
42 | Try to use the network cable to download. The wireless network is unstable and the download speed is affected.
43 | Under the condition that the program does not change the directory configuration, for uncompleted plot, It will continue to transmit the remaining part only.
44 |
45 |
46 |
47 |
48 | for Windows.
49 | For beginners, we provide a simple GUI program. go Windows-GUI
50 | 
51 | If you are familiar with PowerShell, you can use the following scripts.
52 |
53 | #Run it on Windows
54 | #Now, open a powershell window (Click Start, type PowerShell, and then click Windows PowerShell)
55 | #run the follow command(If there is a problem with the execution sequence, please execute line by line)
56 |
57 | #del .\\CloudChiaPlotter.exe
58 |
59 | [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
60 | Invoke-WebRequest -Uri https://github.com/anyplots/cloud-chia-plotter/releases/download/v3/cloud-chia-plotter-v3-win-x64.exe -Outfile .\\CloudChiaPlotter.exe
61 | .\\CloudChiaPlotter.exe -p {your project token} -d "d:\\;e:\\;f:\\" -b {your bandwidth}
62 |
63 |
64 |
65 | for Linux series.
66 |
67 | # run the follow script in the linux console
68 | # Note that different platforms correspond to different link versions
69 |
70 | #rm ./CloudChiaPlotter
71 |
72 | wget -O CloudChiaPlotter https://github.com/anyplots/cloud-chia-plotter/releases/download/v3/cloud-chia-plotter-v3-linux-x64
73 | chmod +x ./CloudChiaPlotter
74 | ./CloudChiaPlotter -p {your project token} -d "/mnt/d/;/mnt/e/;/mnt/f/" -b {your bandwidth}
75 |
76 |
77 |
78 | for Mac OS series.
79 |
80 | #Running the CloudChiaPlotter on Mac OS
81 | #Now, open a terminal window (Mac OS version of the command line)
82 | #run the follow command:
83 |
84 | #rm ./CloudChiaPlotter
85 | curl -LJ https://github.com/anyplots/cloud-chia-plotter/releases/download/v3/cloud-chia-plotter-v3-osx-x64 -o ./CloudChiaPlotter
86 | chmod +x ./CloudChiaPlotter
87 | ./CloudChiaPlotter -p {your project token} -d "/mnt/d/;/mnt/e/;/mnt/f/" -b {your bandwidth}
88 |
89 |
90 |
91 | for Arm64 Linux series(Raspberry Pi 3, Pi 4, etc)
92 |
93 | # run the follow script in the linux console
94 | # Note that different platforms correspond to different link versions
95 |
96 | #rm ./CloudChiaPlotter
97 |
98 | wget -O CloudChiaPlotter https://github.com/anyplots/cloud-chia-plotter/releases/download/v3/cloud-chia-plotter-v3-linux-arm64
99 | chmod +x ./CloudChiaPlotter
100 | ./CloudChiaPlotter -p {your project token} -d "/mnt/d/;/mnt/e/;/mnt/f/" -b {your bandwidth}
101 |
102 |
103 |
104 | for Arm32 Linux series.
105 |
106 | # run the follow script in the linux console
107 | # Note that different platforms correspond to different link versions
108 |
109 | #rm ./CloudChiaPlotter
110 |
111 | wget -O CloudChiaPlotter https://github.com/anyplots/cloud-chia-plotter/releases/download/v3/cloud-chia-plotter-v3-linux-arm32
112 | chmod +x ./CloudChiaPlotter
113 | ./CloudChiaPlotter -p {your project token} -d "/mnt/d/;/mnt/e/;/mnt/f/" -b {your bandwidth}
114 |
115 |
116 | Tested download Speed
117 | Different user experiences may vary, depending on your network and the plotting server you choose.
118 |
119 | download from U.S. based plotting server at the following countries(bandwidth 1 Gbit/s):
120 | US: ping latency ~50ms, download speed > 1000Mbit/s, ~ 13 minutes per plot.
121 | UK: ping latency ~80ms, download speed > 1000Mbit/s. ~ 13 minutes per plot.
122 | DE: ping latency ~100ms, download speed > 1000Mbit/s. ~ 13 minutes per plot.
123 | FR: ping latency ~100ms, download speed > 1000Mbit/s. ~ 13 minutes per plot.
124 | JP: ping latency ~160ms, download speed > 900Mbit/s. ~ 15 minutes per plot.
125 | AU: ping latency ~220ms, download speed > 800Mbit/s. ~ 16 minutes per plot.
126 | IN: ping latency ~280ms, download speed > 700Mbit/s. ~ 18 minutes per plot.
127 |
128 |
129 |
130 |
131 |
132 |
133 | How to build it from source code?
134 |
135 | As this project is based on Microsoft .NET 6.0(more information https://dotnet.microsoft.com/en-us/download/dotnet/6.0)
136 | You can use Microsoft Visual Studio 2022 to quickly compile various system versions
137 |
138 | We have precompiled some executable files for several common OS(see here https://github.com/anyplots/cloud-chia-plotter/releases).
139 | the compile configuration files is located "Properties/PublishProfiles/"
140 | linux-arm.pubxml
141 | linux-arm64.pubxml
142 | linux-x64.pubxml
143 | osx-x64.pubxml
144 | win-x64.pubxml
145 |
146 | You also can compile it for other OS by yourself.
147 | It is not recommended to change the business logic in the code, otherwise you may not be able to download or occurs unknown errors.
148 |
149 |
150 | Build it for linux without VS2022
151 |
152 | git clone https://github.com/anyplots/cloud-chia-plotter
153 | wget -O ./dotnet-install.sh https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.sh
154 | chmod +x ./dotnet-install.sh
155 | ./dotnet-install.sh
156 | ./.dotnet/dotnet publish ./cloud-chia-plotter/CloudChiaPlotter.csproj /p:PublishProfile=./cloud-chia-plotter/Properties/PublishProfiles/linux-x64.pubxml
157 |
158 |
159 |
--------------------------------------------------------------------------------
/PlotWriter.cs:
--------------------------------------------------------------------------------
1 | /* copyright @ anyplots.com, All rights reserved, visit https://anyplots.com/ for more information */
2 | using System;
3 | using System.Collections.Generic;
4 | using System.Text;
5 | using System.Net;
6 | using System.IO;
7 | using System.Threading;
8 | using System.Diagnostics;
9 | namespace anyplots
10 | {
11 | class Block
12 | {
13 | public const long BlockSize = 1024 * 1024 * 4;
14 | public long position = 0;
15 | public int size = 0;
16 | public byte[] buffer;
17 |
18 | public Block(long pos, int size)
19 | {
20 | position = pos;
21 | buffer = new Byte[size];
22 | }
23 | }
24 |
25 | class PlotWriter : IDisposable
26 | {
27 | string dir = "";
28 | string fileid = "";
29 | string filename = "";
30 | Queue queue = new Queue();
31 | SortedList downloaded = new SortedList(1024);
32 | FileStream data = null;
33 | FileStream conf = null;
34 | Thread writer = null;
35 | bool running = true;
36 | public Exception Err = null;
37 | public Int64 FileSize;
38 | public PlotWriter(string dir, string fileid, string filename, Int64 size)
39 | {
40 | this.dir = dir;
41 | this.fileid = fileid;
42 | this.filename = filename;
43 | this.FileSize = size;
44 | try
45 | {
46 | data = new FileStream(dir + fileid + "." + filename + ".data" + ApiController.Ext, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
47 | conf = new FileStream(dir + fileid + "." + filename + ".conf" + ApiController.Ext, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
48 | if (data.Length != size)
49 | {
50 | data.SetLength(size);
51 | data.Position = 0;
52 | }
53 | else
54 | {
55 | data.Position = DownloadedPosition;
56 | }
57 | downloaded.Add(data.Position, true);
58 | SaveProgress(true);
59 | writer = new Thread(Writing);
60 | writer.Start();
61 | ApiController.Logging(false, 0, "Using Dir: " + dir);
62 | }
63 | catch (Exception ex)
64 | {
65 | Dispose();
66 | try { File.Delete(dir + fileid + "." + filename + ".data" + ApiController.Ext); } catch { }
67 | try { File.Delete(dir + fileid + "." + filename + ".conf" + ApiController.Ext); } catch { }
68 | throw new Exception(ex.Message + ex.StackTrace);
69 | }
70 | }
71 | public void Dispose()
72 | {
73 | Err = new Exception("Disposed");
74 | if (running)
75 | {
76 | running = false;
77 | }
78 | if (writer != null)
79 | {
80 | try
81 | {
82 | for (int i = 0; i < 10000 && writer.IsAlive; i++)
83 | {
84 | Thread.Sleep(10);
85 | }
86 | }
87 | catch { };
88 | writer = null;
89 | }
90 | if (conf != null)
91 | {
92 | try { conf.Close(); } catch { }
93 | conf = null;
94 | }
95 | if (data != null)
96 | {
97 | try { data.Close(); } catch { }
98 | data = null;
99 | }
100 | }
101 | public long DownloadedPosition
102 | {
103 | get {
104 | Byte[] bytes_ = new byte[12];
105 | conf.Position = 0;
106 | if (conf.Read(bytes_, 0, 12) == 12)
107 | {
108 | Int64 len = BitConverter.ToInt64(bytes_, 0);
109 | if (len.GetHashCode() == BitConverter.ToInt32(bytes_, 8))
110 | {
111 | return len;
112 | }
113 | else
114 | {
115 | ApiController.Logging(false,0,"Hash Error");
116 | return 0;
117 | }
118 | }
119 | return 0;
120 | }
121 | }
122 | void SaveProgress(bool changed)
123 | {
124 | while (downloaded.Count > 1)
125 | {
126 | if (downloaded.Keys[0] + Block.BlockSize == downloaded.Keys[1])
127 | {
128 | downloaded.RemoveAt(0);
129 | changed = true;
130 | }
131 | else
132 | {
133 | break;
134 | }
135 | }
136 | if (changed && downloaded.Count > 0)
137 | {
138 | conf.Position = 0;
139 | conf.Write(BitConverter.GetBytes(downloaded.Keys[0]), 0, 8);
140 | conf.Write(BitConverter.GetBytes(downloaded.Keys[0].GetHashCode()), 0, 4);
141 | conf.Flush();
142 | }
143 | }
144 | public double Percent = 0;
145 | public bool IsCompleted = false;
146 |
147 | public bool IsQueueFull
148 | {
149 | get
150 | {
151 | lock (queue)
152 | {
153 | return queue.Count > 128;
154 | }
155 | }
156 | }
157 | public void Write(Block block)
158 | {;
159 | if (Err != null) { return; }
160 | if (!running) { Err = new Exception("writing stopped!!!"); return; }
161 | lock (queue)
162 | {
163 | queue.Enqueue(block);
164 | }
165 | }
166 | public void Finish()
167 | {
168 | ApiController.Logging(false,0,"finishing ...");
169 | if (Err != null || !IsCompleted)
170 | {
171 | throw new Exception("error, plot not completed !!!");
172 | }
173 | Dispose();
174 | File.Move(dir + fileid + "." + filename + ".data" + ApiController.Ext, dir + filename);
175 | if (File.Exists(dir + filename))
176 | {
177 | File.Delete(dir + fileid + "." + filename + ".conf" + ApiController.Ext);
178 | }
179 | else
180 | {
181 | throw new Exception("rename file failed!!!");
182 | }
183 | }
184 | void Writing(object o)
185 | {
186 | try
187 | {
188 | int waits = 0;
189 | int writed = 0;
190 | DateTime nextflush = DateTime.Now.AddSeconds(5);
191 | while (running)
192 | {
193 |
194 | List blocks = new List(200);
195 | lock (queue)
196 | {
197 | if (queue.Count > 16 || waits > 6)
198 | {
199 | while (queue.Count > 0)
200 | {
201 | blocks.Add(queue.Dequeue());
202 | }
203 | waits = 0;
204 | }
205 | else
206 | {
207 | waits++;
208 | }
209 | }
210 | if (!running) return;
211 | if (blocks.Count > 0)
212 | {
213 | blocks.Sort(delegate (Block a, Block b) { return a.position.CompareTo(b.position); });
214 | foreach (Block block in blocks)
215 | {
216 | if (!running) return;
217 | if (block.position < downloaded.Keys[0]) continue;
218 | data.Position = block.position;
219 | data.Write(block.buffer, 0, block.size);
220 | downloaded.Add(data.Position, true);
221 | writed++;
222 | }
223 | }
224 | else
225 | {
226 | Thread.Sleep(5);
227 | }
228 | if (writed > 0 && nextflush < DateTime.Now)
229 | {
230 | data.Flush();
231 | SaveProgress(false);
232 | writed = 0;
233 | nextflush = DateTime.Now.AddSeconds(5);
234 | }
235 | Percent = Math.Min(99.9999,(downloaded.Keys[0] + (downloaded.Count - 1) * Block.BlockSize) * 100d / FileSize);
236 | if(downloaded.Keys[downloaded.Count - 1] == FileSize)
237 | {
238 | IsCompleted = downloaded.Keys[downloaded.Count - 1] - downloaded.Keys[0] < Block.BlockSize;
239 | if (IsCompleted)
240 | {
241 | data.Flush();
242 | }
243 | }
244 | GC.Collect();
245 | }
246 | }
247 | catch(Exception ex)
248 | {
249 | Err = ex;
250 | }
251 | running = false;
252 | }
253 | }
254 | }
255 |
--------------------------------------------------------------------------------
/BlockTransfer.cs:
--------------------------------------------------------------------------------
1 | /* copyright @ anyplots.com, All rights reserved, visit https://anyplots.com/ for more information */
2 | using System;
3 | using System.Collections.Generic;
4 | using System.Text;
5 | using System.Net;
6 | using System.IO;
7 | using System.Threading;
8 | using System.Diagnostics;
9 | using Newtonsoft.Json;
10 | using System.Net.Sockets;
11 | namespace anyplots
12 | {
13 | static class BlockTransfer
14 | {
15 | static Int32 ReadInt32(byte[] buffer, int offset) //prevent Big-Endian, Little-Endian problem
16 | {
17 | return Convert.ToInt32(Encoding.ASCII.GetString(buffer, offset, 8), 16);
18 | }
19 | static Byte[] Int32Bytes(int v) //prevent Big-Endian, Little-Endian problem
20 | {
21 | return Encoding.ASCII.GetBytes(v.ToString("x8"));
22 | }
23 | static Queue> stats = new Queue>(100000);
24 | static Int64 counter = 0;
25 | public static float Speed
26 | {
27 | get
28 | {
29 | lock (stats)
30 | {
31 | Int64 ticks = DateTime.Now.Ticks - 3000000000;
32 | while (stats.Count > 0 && stats.Peek().Key < ticks)
33 | {
34 | stats.Dequeue();
35 | }
36 | if (stats.Count > 0)
37 | {
38 | float taken = (DateTime.Now.Ticks - stats.Peek().Key) / 10000000.0f;
39 | float speed = (float)((counter - stats.Peek().Value) * 8 / taken ) / 1024 / 1024;
40 | return speed;
41 | }
42 | }
43 | return 0f;
44 | }
45 | }
46 | public static bool GetBlock(string url, Block block)
47 | {
48 | block.size = 0;
49 | using (Socket socket = new Socket(SocketType.Stream, ProtocolType.Tcp))
50 | {
51 |
52 | try
53 | {
54 | socket.NoDelay = true;
55 | socket.ReceiveTimeout = 300000;
56 | socket.SendTimeout = 1000;
57 | socket.ReceiveBufferSize = 6 * (1 << 20) + 1024;
58 | string[] items = url.Split('/');
59 | IPAddress ip = IPAddress.Parse(items[0].Split(':')[0]);
60 | Int32 port = Int32.Parse(items[0].Split(':')[1]);
61 | IAsyncResult ar = socket.BeginConnect(ip, port, null, null);
62 | if (ar.AsyncWaitHandle.WaitOne(10000))
63 | {
64 | socket.EndConnect(ar);
65 | if (socket.Connected)
66 | {
67 | string request = "GET\n" + items[1] + "\n" + block.position.ToString() + "\n" + (block.position + block.buffer.Length - 1).ToString() + "\n";
68 | Byte[] data_ = Encoding.ASCII.GetBytes(request.PadRight(248, ' '));
69 | if (socket.Send(data_) != 248)
70 | {
71 | ApiController.Logging(false, 0, "send request error!");
72 | return false;
73 | }
74 | if (socket.Send(Int32Bytes(CRC(data_, data_.Length))) != 8)
75 | {
76 | ApiController.Logging(false, 0, "send crc error!");
77 | return false;
78 |
79 | }
80 | int received = 0;
81 | while (received < block.buffer.Length)
82 | {
83 | int ret = socket.Receive(block.buffer, received, block.buffer.Length - received, SocketFlags.None);
84 | if (ret > 0)
85 | {
86 | received += ret;
87 | lock (stats)
88 | {
89 | counter += ret;
90 | stats.Enqueue(new KeyValuePair(DateTime.Now.Ticks, counter));
91 | while (stats.Count > 100000)
92 | {
93 | stats.Dequeue();
94 | }
95 | }
96 | if(received == 8 &&
97 | block.buffer[0] == (byte)'7' &&
98 | block.buffer[1] == (byte)'f' &&
99 | block.buffer[2] == (byte)'f' &&
100 | block.buffer[3] == (byte)'f' &&
101 | block.buffer[4] == (byte)'f' &&
102 | block.buffer[5] == (byte)'f' &&
103 | block.buffer[6] == (byte)'f' &&
104 | block.buffer[7] == (byte)'f')
105 | {
106 | return false;
107 | }
108 | }
109 | else
110 | {
111 | return false;
112 | }
113 | }
114 | Byte[] crc = new byte[8];
115 | if (socket.Receive(crc) != 8 || CRC(block.buffer, received) != ReadInt32(crc, 0))
116 | {
117 | ApiController.Logging(false, 0, "crc error!");
118 | return false;
119 | }
120 | block.size = block.buffer.Length;
121 | return true;
122 | }
123 | }
124 | }
125 | catch (SocketException ex)
126 | {
127 | if(ex.SocketErrorCode == SocketError.ConnectionReset || ex.SocketErrorCode == SocketError.TimedOut)
128 | {
129 | return false;
130 | }
131 | ApiController.Logging(false, 0, ex.SocketErrorCode + "\r\n" + ex.Message + "\r\n" + ex.StackTrace);
132 | }
133 | catch (Exception ex)
134 | {
135 | ApiController.Logging(false,0,ex.Message + "\r\n" + ex.StackTrace);
136 | }
137 | finally
138 | {
139 | try { socket.Shutdown(SocketShutdown.Both); } catch { }
140 | }
141 | }
142 | return false;
143 | }
144 |
145 | private const UInt32 magic = 0xedb88320u;
146 |
147 | private static UInt32[] table = new UInt32[16 * 256];
148 |
149 | static BlockTransfer()
150 | {
151 | for (UInt32 i = 0; i < 256; i++)
152 | {
153 | UInt32 res = i;
154 | for (int t = 0; t < 16; t++)
155 | {
156 | for (int k = 0; k < 8; k++) res = (res & 1) == 1 ? magic ^ (res >> 1) : (res >> 1);
157 | table[(t * 256) + i] = res;
158 | }
159 | }
160 | }
161 | public static Int32 CRC(byte[] buffer, int length)
162 | {
163 | UInt32 value = 0xffffffff;
164 | int offset = 0;
165 | while (length >= 16)
166 | {
167 | var a = table[(3 * 256) + buffer[offset + 12]]
168 | ^ table[(2 * 256) + buffer[offset + 13]]
169 | ^ table[(1 * 256) + buffer[offset + 14]]
170 | ^ table[(0 * 256) + buffer[offset + 15]];
171 |
172 | var b = table[(7 * 256) + buffer[offset + 8]]
173 | ^ table[(6 * 256) + buffer[offset + 9]]
174 | ^ table[(5 * 256) + buffer[offset + 10]]
175 | ^ table[(4 * 256) + buffer[offset + 11]];
176 |
177 | var c = table[(11 * 256) + buffer[offset + 4]]
178 | ^ table[(10 * 256) + buffer[offset + 5]]
179 | ^ table[(9 * 256) + buffer[offset + 6]]
180 | ^ table[(8 * 256) + buffer[offset + 7]];
181 |
182 | var d = table[(15 * 256) + ((byte)value ^ buffer[offset])]
183 | ^ table[(14 * 256) + ((byte)(value >> 8) ^ buffer[offset + 1])]
184 | ^ table[(13 * 256) + ((byte)(value >> 16) ^ buffer[offset + 2])]
185 | ^ table[(12 * 256) + ((value >> 24) ^ buffer[offset + 3])];
186 |
187 | value = d ^ c ^ b ^ a;
188 | offset += 16;
189 | length -= 16;
190 | }
191 |
192 | while (--length >= 0)
193 | value = table[(byte)(value ^ buffer[offset++])] ^ value >> 8;
194 |
195 | return (Int32)(value & 0x7fffffff);
196 | }
197 | static string MD5(string content)
198 | {
199 | using (var md5 = System.Security.Cryptography.MD5.Create())
200 | {
201 | var result = md5.ComputeHash(Encoding.UTF8.GetBytes(content));
202 | return BitConverter.ToString(result).Replace("-", "");
203 | }
204 | }
205 | }
206 |
207 | }
208 |
--------------------------------------------------------------------------------
/Client.cs:
--------------------------------------------------------------------------------
1 | /* copyright @ anyplots.com, All rights reserved, visit https://anyplots.com/ for more information */
2 | using System;
3 | using System.Collections.Generic;
4 | using System.Text;
5 | using System.Net;
6 | using System.Net.NetworkInformation;
7 | using System.IO;
8 | using System.Threading;
9 | using System.Diagnostics;
10 | namespace anyplots
11 | {
12 | class Client
13 | {
14 | public static String downloadstatus = "";
15 | public static int optimizedmaxthreads = -1;
16 | string url = "";
17 | int id = 0;
18 | PlotWriter writer = null;
19 | bool running = true;
20 | static int maxthreads = 8;
21 | List threads = new List();
22 |
23 | bool eof = false;
24 | long position = 0;
25 | long endposition = 0;
26 | object syncposition = new object();
27 | Block NextBlock()
28 | {
29 | Block block = null;
30 | lock (syncposition)
31 | {
32 | try
33 | {
34 | if (endposition > position)
35 | {
36 | block = new Block(position, (int)Math.Min(Block.BlockSize, endposition - position));
37 | return block;
38 | }
39 | else
40 | {
41 | eof = true;
42 | return null;// end
43 | }
44 | }
45 | finally
46 | {
47 | if (block != null) position += block.buffer.Length;
48 | }
49 | }
50 | }
51 | public Client(string url, int id, PlotWriter writer)
52 | {
53 | this.url = url;
54 | this.id = id;
55 | this.writer = writer;
56 | this.endposition = writer.FileSize;
57 | this.position = writer.DownloadedPosition;
58 | }
59 | bool diskslow = false;
60 | void DownloadThread(object o)
61 | {
62 | int tid = (int)o;
63 | Block block = null;
64 | Stopwatch sw = new Stopwatch();
65 | Random rnd = new Random();
66 | while (running)
67 | {
68 | try
69 | {
70 | if (block == null)
71 | {
72 | if (tid >= maxthreads)
73 | {
74 | Thread.Sleep(rnd.Next(5, 15));
75 | if (eof)
76 | {
77 | return; // end
78 | }
79 | continue;
80 | }
81 | }
82 | for (int i = 0; running && i < 100000; i++)
83 | {
84 | if (!writer.IsQueueFull) break;
85 | if (i > 1000) { diskslow = true; }
86 | Thread.Sleep(10);
87 | }
88 | diskslow = false;
89 | if (block == null)
90 | {
91 | block = NextBlock();
92 | }
93 | if (block == null)
94 | {
95 | return; // end
96 | }
97 | sw.Restart();
98 | sw.Start();
99 | bool ret = BlockTransfer.GetBlock(url, block);
100 | sw.Stop();
101 | if (ret)
102 | {
103 | writer.Write(block);
104 | block = null;
105 | }
106 | else
107 | {
108 | Thread.Sleep(5000);
109 | }
110 | }
111 | catch (Exception ex)
112 | {
113 | ApiController.Logging(true, id, "DownloadThread(" + tid + "):" + ex.Message + "\r\n" + ex.StackTrace);
114 | Thread.Sleep(10000);
115 | }
116 | }
117 | }
118 | int pingavg = 100;
119 | int pingmin = 2000;
120 | int pinglos = 0;
121 | void PingServer(object o)
122 | {
123 | IPAddress ip = IPAddress.Parse((string)o);
124 | Queue items = new Queue(500);
125 | Stopwatch watch = new Stopwatch();
126 | while (running)
127 | {
128 | watch.Reset();
129 | watch.Start();
130 | using (Ping ping = new Ping())
131 | {
132 | try
133 | {
134 | Byte[] data = Encoding.ASCII.GetBytes((Guid.NewGuid().ToString() + Guid.NewGuid().ToString() + Guid.NewGuid().ToString()).Substring(0, 64));
135 | PingReply pr = ping.Send(ip, 1000, data);
136 | if (pr.Status == IPStatus.Success)
137 | {
138 | items.Enqueue((int)pr.RoundtripTime);
139 | }
140 | else
141 | {
142 | items.Enqueue(1000);
143 | }
144 | }
145 | catch { items.Enqueue(1000); }
146 | }
147 | Queue items_ = new Queue(500);
148 | while (items.Count > 300) items.Dequeue();
149 | int total = 0,count = 0, min = 1000;
150 | while (items.Count > 0)
151 | {
152 | int t = items.Dequeue();
153 | total += t;
154 | min = Math.Min(min, t);
155 | if (t >= 1000) count++;
156 | items_.Enqueue(t);
157 | }
158 | items = items_;
159 | pingmin = min;
160 | pingavg = total / items_.Count;
161 | pinglos = 100 * count / items_.Count;
162 | if (optimizedmaxthreads == -1)
163 | {
164 | float max_;
165 | if (items_.Count > 200 && pinglos > 0)
166 | {
167 | max_ = Math.Min(BlockTransfer.Speed * 5, ApiController.ClientBandwidth) / 16f * Math.Min(8, Math.Max(1, (pingavg / 50f)));
168 | }
169 | else
170 | {
171 | max_ = ApiController.ClientBandwidth / 16f * Math.Min(8, Math.Max(1, (pingavg / 50f)));
172 | }
173 | max_ -= max_ * pinglos * 2 / 100;
174 | if (max_ < 4)
175 | {
176 | max_ = 4;
177 | }
178 | if (max_ > threads.Count)
179 | {
180 | max_ = threads.Count;
181 | }
182 | maxthreads = (int)max_;
183 | }
184 | watch.Stop();
185 | if (watch.ElapsedMilliseconds < 995)
186 | {
187 | Thread.Sleep(1000 - (int)watch.ElapsedMilliseconds);
188 | }
189 | }
190 | }
191 | public bool Run()
192 | {
193 | bool ret = true;
194 | try
195 | {
196 | maxthreads = ApiController.ClientBandwidth / 16;
197 | if (maxthreads > 128) maxthreads = 128;
198 | if (maxthreads < 4) maxthreads = 4;
199 | Thread t = new Thread(PingServer);
200 | t.Start(url.Split(':')[0]);
201 | for (int i = 0; i < 128; i++)
202 | {
203 | threads.Add(new Thread(DownloadThread));
204 | threads[i].Start(i);
205 | Thread.Sleep(10);
206 | }
207 | try { if (Console.WindowWidth < 80) Console.WindowWidth = 80; } catch { }
208 | DateTime nextstat = DateTime.MinValue;
209 | string name = url.Split('/')[1].Split('.')[1];
210 | while (running)
211 | {
212 | bool isalive = false;
213 | for (int i = 0; i < threads.Count; i++)
214 | {
215 | if (threads[i].IsAlive)
216 | {
217 | isalive = true;
218 | break;
219 | }
220 | }
221 | if (writer.Err != null)
222 | {
223 | ret = false;
224 | running = false;
225 | writer.Dispose();
226 | ApiController.Logging(true, id, writer.Err.Message + "\r\n" + writer.Err.StackTrace);
227 | break;
228 | }
229 | if (isalive)
230 | {
231 | double percent = Math.Max(writer.Percent - 0.0001f, 0); // 100% will cause plot delete on the server side.
232 | float speed = BlockTransfer.Speed;
233 | Console.Write(("\r[" + id + "] " + percent.ToString("f4") + "%, " + speed.ToString("f1") + "Mbps, " + maxthreads + " Threads").PadRight(45, ' ') +
234 | (diskslow ? "write queue full, disk I/O speed is slow" : (name.Substring(0, 30) + "...")));
235 |
236 | if (optimizedmaxthreads != -1)
237 | {
238 | maxthreads = optimizedmaxthreads;
239 | }
240 | if (nextstat < DateTime.Now)
241 | {
242 | nextstat = DateTime.Now.AddSeconds(30);
243 | int status = ApiController.Stats(id, (float)percent, (float)speed, pingmin, pingavg, pinglos);
244 | if (status == -9999) // error
245 | {
246 | running = false;
247 | ret = false;
248 | writer.Dispose();
249 | break;
250 | }
251 | else if (status < 0)
252 | {
253 | optimizedmaxthreads = -status;
254 | }
255 | }
256 | }
257 | else
258 | {
259 | if (writer.IsCompleted)
260 | {
261 | writer.Finish();
262 | ApiController.Stats(id, 100, 0, pingmin, pingavg, pinglos);
263 | break;
264 | }
265 | }
266 | Thread.Sleep(1000);
267 | }
268 | for (int i = 0; i < threads.Count; i++)
269 | {
270 | threads[i].Join();
271 | }
272 | }
273 | finally
274 | {
275 | running = false;
276 | try { writer.Dispose(); } catch { }
277 | }
278 | return ret;
279 | }
280 | }
281 | }
282 |
--------------------------------------------------------------------------------
/Program.cs:
--------------------------------------------------------------------------------
1 | /* copyright @ anyplots.com, All rights reserved, visit https://anyplots.com/ for more information */
2 | using System;
3 | using System.Threading;
4 | using System.IO;
5 | using System.Collections.Generic;
6 | using System.Diagnostics;
7 | using System.Reflection;
8 | namespace anyplots
9 | {
10 | class Program
11 | {
12 | static string ver = null;
13 | public static string Version
14 | {
15 | get
16 | {
17 | if (ver == null)
18 | {
19 | try
20 | {
21 | using (Process p = Process.GetCurrentProcess())
22 | {
23 | string path = p.MainModule.FileName;
24 | using (FileStream fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
25 | {
26 | byte[] bytes = new byte[fs.Length];
27 | fs.Read(bytes, 0, bytes.Length);
28 | int crc = BlockTransfer.CRC(bytes, bytes.Length);
29 | ver = "2022021801@" + crc.ToString("x8");
30 | }
31 | }
32 | }
33 | catch { ver = "-"; }
34 | }
35 | return ver;
36 | }
37 | }
38 | static int nextdir = 0;
39 | public static List Dirs = new List();
40 | public static string NextDir
41 | {
42 | get { return Dirs[nextdir++ % Dirs.Count]; }
43 | }
44 |
45 | static void MyHandler(object sender, UnhandledExceptionEventArgs args)
46 | {
47 | try
48 | {
49 | if (args.ExceptionObject != null)
50 | {
51 | Exception e = (Exception)args.ExceptionObject;
52 | ApiController.Logging(false, 0, "Error caught : " + e.Message);
53 | ApiController.Logging(true, 0, "MyHandler1:" + args.IsTerminating + "\r\n" + e.Message + "\r\n" + e.StackTrace);
54 | }
55 | else
56 | {
57 |
58 | ApiController.Logging(true, 0, "MyHandler2:" + args.IsTerminating);
59 | }
60 | }
61 | catch { }
62 | }
63 |
64 | static FileStream fslock = null;
65 | static FileStream LockFile(string file)
66 | {
67 | try
68 | {
69 | FileStream fslock = new FileStream(file, FileMode.Create, FileAccess.ReadWrite, FileShare.None);
70 | try
71 | {
72 | fslock.Write(System.Text.Encoding.ASCII.GetBytes("lock"), 0, 4);
73 | return fslock;
74 | }
75 | catch { fslock.Close(); }
76 | }
77 | catch { }
78 | return null;
79 | }
80 | static void InitClient()
81 | {
82 | foreach (string file in Directory.GetFiles(AppDomain.CurrentDomain.BaseDirectory, "*.client"))
83 | {
84 | FileStream lock_ = LockFile(file);
85 | if (lock_ != null)
86 | {
87 | string token = Path.GetFileNameWithoutExtension(file);
88 | if (token != ApiController.Login(ApiController.ProjectToken, token))
89 | {
90 | lock_.Close();
91 | File.Delete(file);
92 | }
93 | else
94 | {
95 | fslock = lock_;
96 | ApiController.ClientToken = token;
97 | return;
98 | }
99 | }
100 | Thread.Sleep(5000);
101 | }
102 | string newtoken_ = ApiController.Login(ApiController.ProjectToken, "");
103 | if (newtoken_.Length == 40)
104 | {
105 | FileStream lock_ = LockFile(AppDomain.CurrentDomain.BaseDirectory + newtoken_ + ".client");
106 | if (lock_ == null)
107 | {
108 | throw new Exception("unkown error");
109 | }
110 | fslock = lock_;
111 | ApiController.ClientToken = newtoken_;
112 | }
113 | else
114 | {
115 | throw new Exception("login failed, unkown token: " + newtoken_);
116 | }
117 | }
118 | static bool IsWindows
119 | {
120 | get { return AppDomain.CurrentDomain.BaseDirectory.Contains('\\'); }
121 | }
122 | static string NormalizeDirPath(string path)
123 | {
124 | if (IsWindows)
125 | {
126 | return path.TrimEnd(new char[] { '/', '\\' }) + "\\";
127 | }
128 | else
129 | {
130 | return path.TrimEnd(new char[] { '/', '\\' }) + "/";
131 | }
132 | }
133 | static void Main(string[] args)
134 | {
135 | try
136 | {
137 | AppDomain.CurrentDomain.UnhandledException += new UnhandledExceptionEventHandler(MyHandler);
138 | }
139 | catch { }
140 | if (args.Length < 6)
141 | {
142 | Console.WriteLine("Version: " + Version);
143 | Console.WriteLine("Usage: ");
144 | if (IsWindows)
145 | {
146 | Console.WriteLine("use the command like: .\\CloudChiaPlotter.exe -p {project_token} -d d:\\;e:\\;f:\\;g:\\ -b 1000");
147 | Console.WriteLine("-p {project_token} such as: -p 000001236c46bb1d0cddf079c7c345ea5bd727e28c4");
148 | Console.WriteLine("-d save the chia plots to these folders, such as: -d d:\\;e:\\;f:\\;g:\\");
149 | }
150 | else
151 | {
152 | Console.WriteLine("use the command like: ./CloudChiaPlotter -p {project_token} -d /mnt/d;/mnt/e/;/mnt/f;/mnt/g -b 1000");
153 | Console.WriteLine("-p {project_token} such as: -p 000001236c46bb1d0cddf079c7c345ea5bd727e28c4");
154 | Console.WriteLine("-d save the chia plots to these folders, such as: -d /mnt/d;/mnt/e/;/mnt/f;/mnt/g");
155 | }
156 | Console.WriteLine("-b your network bandwidth, such as: -b 1000 for 1000Mbps");
157 | Console.WriteLine(" if you don't know it, please visit and test at https://speedtest.net");
158 | Console.WriteLine("\r\nPress any key to continue");
159 | Console.ReadKey();
160 | return;
161 | }
162 | else
163 | {
164 | try
165 | {
166 | for (int i = 0; i < args.Length; i++)
167 | {
168 | switch (args[i])
169 | {
170 | case "-p":
171 | ApiController.ProjectToken = args[++i];
172 | break;
173 | case "-d":
174 | Dictionary filter = new Dictionary();
175 | foreach (string item in args[++i].Split(';'))
176 | {
177 | string dir = NormalizeDirPath(item);
178 | if (Directory.Exists(dir))
179 | {
180 | if (!filter.ContainsKey(dir.ToLower()))
181 | {
182 | if (TestDir(dir))
183 | {
184 | filter.Add(dir.ToLower(), null);
185 | Dirs.Add(dir);
186 | ApiController.Logging(false, 0, "add dir:" + dir);
187 | }
188 | else
189 | {
190 | ApiController.Logging(false, 0, "test dir failed, skip dir:" + dir);
191 | }
192 | }
193 | else
194 | {
195 | ApiController.Logging(false, 0, "skip duplicate dir:" + dir);
196 | }
197 | }
198 | else
199 | {
200 | ApiController.Logging(false, 0, "unkown dir:" + dir);
201 | }
202 | }
203 | break;
204 | case "-b":
205 | ApiController.ClientBandwidth = int.Parse(args[++i]);
206 | break;
207 | default:
208 | ApiController.Logging(false, 0, "unkown argument:" + args[i]);
209 | break;
210 | }
211 | }
212 | if (ApiController.ProjectToken.Length != 40)
213 | {
214 | ApiController.Logging(false, 0, "Project_token is incorrect:" + ApiController.ProjectToken);
215 | throw new Exception("Project_token is incorrect:" + ApiController.ProjectToken);
216 | }
217 | if (Dirs.Count == 0)
218 | {
219 | ApiController.Logging(false, 0, "No available directories");
220 | throw new Exception("No available directories");
221 | }
222 | if (ApiController.ClientBandwidth == 0)
223 | {
224 | ApiController.Logging(false, 0, "Bandwidth not set");
225 | throw new Exception("Bandwidth not set");
226 | }
227 | }
228 | catch (Exception ex)
229 | {
230 |
231 | Console.WriteLine("Bad arguments!" + ex.Message);
232 | Console.WriteLine("Press any key to continue");
233 | Console.ReadKey();
234 | return;
235 | }
236 | }
237 | InitClient();
238 | ApiController.Logging(false, 0, "client token:" + ApiController.ClientToken);
239 | Run();
240 | }
241 | static long GetFreeSpace(string path)
242 | {
243 | try
244 | {
245 | return new DriveInfo(path).AvailableFreeSpace;
246 | }
247 | catch (Exception ex)
248 | {
249 | ApiController.Logging(false, 0, ex.Message);
250 | }
251 | return 0;
252 | }
253 | public static int GetFreeSpace(out int cnt)
254 | {
255 | int total = 0;
256 | cnt = 0;
257 | foreach (string dir in Program.Dirs)
258 | {
259 | try
260 | {
261 | cnt++;
262 | int ret = (int)(GetFreeSpace(dir) >> 30);
263 | if (ret > 1000000000) { ret = 0; } // error
264 | total += ret;
265 | }
266 | catch { }
267 | }
268 | return total;
269 | }
270 | static bool TestDir(string path)
271 | {
272 | try
273 | {
274 | using (FileStream fs = new FileStream(path + "testfile", FileMode.Create, FileAccess.Write, FileShare.Read))
275 | {
276 | fs.WriteByte(0x0a);
277 | }
278 | }
279 | catch (Exception ex)
280 | {
281 | ApiController.Logging(false, 0, ex.Message + ex.StackTrace);
282 | return false;
283 | }
284 | finally
285 | {
286 | try { File.Delete(path + "testfile"); } catch { }
287 | }
288 | return true;
289 | }
290 | static Int64 GetFileSize(string file)
291 | {
292 | using (FileStream fs = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
293 | {
294 | return fs.Length;
295 | }
296 | }
297 | static string GetUncompletedPlot(out int fileid, out Int64 filesize, out string filename)
298 | {
299 | fileid = 0;
300 | filesize = 0;
301 | filename = "";
302 | foreach (string dir in Program.Dirs)
303 | {
304 | try
305 | {
306 | foreach (string file in Directory.GetFiles(dir, "*.data" + ApiController.Ext))
307 | {
308 | filesize = GetFileSize(file);
309 | string[] parts = Path.GetFileNameWithoutExtension(file).Split('.');
310 | fileid = int.Parse(Path.GetFileNameWithoutExtension(parts[0]));
311 | filename = parts[1] + ".plot";
312 | return file;
313 | }
314 | }
315 | catch (Exception ex)
316 | {
317 | fileid = 0;
318 | filesize = 0;
319 | ApiController.Logging(true, 0, "GetUncompletedPlot: " + ex.Message + "\r\n" + dir);
320 | }
321 | }
322 | return "";
323 | }
324 | static void Clear(string path)
325 | {
326 | if (File.Exists(path))
327 | {
328 | try
329 | {
330 | ApiController.Logging(false, 0, "Delete: " + path);
331 | File.Delete(path);
332 | }
333 | catch { }
334 | }
335 | path = path.Replace(".data" + ApiController.ClientToken, ".conf" + ApiController.ClientToken);
336 | if (File.Exists(path))
337 | {
338 | try
339 | {
340 | ApiController.Logging(false, 0, "Delete: " + path);
341 | File.Delete(path);
342 | }
343 | catch { }
344 | }
345 | }
346 | static PlotWriter GetPlotWriter(int fileid, string filename, long filesize)
347 | {
348 | for (int i = 0; i < Program.Dirs.Count; i++)
349 | {
350 | string dir = Program.NextDir;
351 | try
352 | {
353 | if (GetFreeSpace(dir) > filesize || File.Exists(dir + fileid + "." + filename + ".data" + ApiController.Ext))
354 | {
355 | return new PlotWriter(dir, fileid.ToString(), filename, filesize);
356 | }
357 | else
358 | {
359 | ApiController.Logging(true, fileid, "Skip No space: " + dir);
360 | }
361 | }
362 | catch (Exception ex)
363 | {
364 | ApiController.Logging(true, fileid, dir + "\r\n" + ex.Message + "\r\n" + ex.StackTrace);
365 | }
366 | }
367 | ApiController.Logging(true, fileid, "No space");
368 | return null;
369 | }
370 | static void Run()
371 | {
372 | string lastname = "";
373 | int fileid = 0;
374 | int errors = 0;
375 | while (true)
376 | {
377 | try
378 | {
379 | fileid = 0;
380 | Dictionary plots = new Dictionary(10000); //check duplicate plot file
381 | foreach (string dir in Program.Dirs)
382 | {
383 | try
384 | {
385 | foreach (string file in Directory.GetFiles(dir, "*.plot"))
386 | {
387 | string key = Path.GetFileName(file);
388 | if (!plots.ContainsKey(key))
389 | {
390 | plots.Add(key, null);
391 | }
392 | }
393 | }
394 | catch { }
395 | }
396 | int uncompletedid = 0;
397 | long filesize;
398 | string filename;
399 | string path = GetUncompletedPlot(out uncompletedid, out filesize, out filename);
400 | fileid = uncompletedid;
401 | string url = ApiController.GetPlot(ref fileid, ref filesize, ref filename);
402 | if (uncompletedid > 0 && uncompletedid != fileid)
403 | {
404 | Clear(path);
405 | uncompletedid = 0;
406 | }
407 | if (fileid > 0)
408 | {
409 | ApiController.Logging(false, 0, "**************************************");
410 | if (lastname == filename || plots.ContainsKey(filename))
411 | {
412 | ApiController.Stats(fileid, 100, 0, -1, -1, -1);
413 | ApiController.Logging(true, fileid, "Skip same file: " + filename);
414 | }
415 | else
416 | {
417 | ApiController.Logging(false, fileid, "Start downloading: " + filename);
418 | using (PlotWriter writer = GetPlotWriter(fileid, filename, filesize))
419 | {
420 | if (writer == null)
421 | {
422 | ApiController.Logging(true, fileid, "Download Failed: " + filename + ", it will auto try again in 300 seconds.");
423 | Thread.Sleep(300000);
424 | continue;
425 | }
426 | Client mc = new Client(url, fileid, writer);
427 | if (mc.Run())
428 | {
429 | errors = 0;
430 | lastname = filename;
431 | ApiController.Logging(true, fileid, "Download Success: " + filename);
432 | }
433 | else
434 | {
435 | errors++;
436 | if (errors < 5)
437 | {
438 | ApiController.Logging(true, fileid, "Download Failed: " + filename + ", it will auto try again in 30 seconds.");
439 | Thread.Sleep(30000);
440 | }
441 | else
442 | {
443 | ApiController.Logging(true, fileid, "Download Failed: " + filename + ", it will auto try again in 300 seconds.");
444 | Thread.Sleep(300000);
445 | }
446 | }
447 | }
448 | }
449 | }
450 | else
451 | {
452 | for (int i = 0; i < 30; i++)
453 | {
454 | Console.Write("\r[" + DateTime.Now.ToString("HH:mm:ss") + "]" + filename);
455 | Thread.Sleep(1000);
456 | }
457 | }
458 | }
459 | catch (Exception ex)
460 | {
461 | ApiController.Logging(true, fileid, "Client.Run:" + ex.Message + ex.StackTrace);
462 | Thread.Sleep(30000);
463 | }
464 | }
465 |
466 |
467 | }
468 | }
469 | }
--------------------------------------------------------------------------------
5 | 
8 | 
11 | 
14 |