├── README.md
├── dfx.json
└── src
    ├── Bucket-HTTP
        ├── Bucket-HTTP.mo
        └── example.mo
    └── Bucket
        ├── Bucket.mo
        └── example.mo


/README.md:
--------------------------------------------------------------------------------
  1 | # Bucket
  2 | 
  3 | The library is a **storage library** for canisters to manage Stable Memory. 
  4 | 
  5 | As far as we know, canisters that storage data into stable memory have many advantages, such as :
  6 | - upgradable (when rts memory goes large)
  7 | - larger storage space : Stable Memory can be allocated to 8 GB as present
  8 | - no GC cost
  9 | 
 10 | Therefore, In order to be compatible with the existing development ecology, we develop two versions :
 11 | 
 12 | - [Bucket](#Bucket)
 13 | - [Bucket-HTTP](#Bucket-HTTP)
 14 | 
 15 | You can use this as simple as using the TireMap.
 16 | 
 17 | <span id="Bucket"></span>
 18 | 
 19 | ##  Bucket
 20 | 
 21 | <span id="prework"></span>
 22 | 
 23 | - First, you need to import Bucket in your project 
 24 | 
 25 |    ```motoko
 26 |    import Bucket "Bucket";
 27 |    ```
 28 | 
 29 | - Second, you need to declare a Bucket
 30 | 
 31 |    **upgrade** ：This means that you can upgrade your canister without discarding files stored in the stablememory. Meanwhile available stablememory will become **aval_stablememory** = **8G - heap memory**
 32 |    
 33 |    ```motoko
 34 |    let bucket = Bucket.Bucket(true); // true : upgradable, false : unupgradable
 35 |    ```
 36 | 
 37 |    You have a few more things to do below: 
 38 | 
 39 |    1. you should use a stable entries to store your key-value pairs during upgrade
 40 | 
 41 |       ```motoko
 42 |       stable var bucket_entries: [(Text,[(Nat64, Nat)])] = [];
 43 |       ```
 44 | 
 45 |    2. You also need to configure the system function [preupgrade and postupgrade](https://smartcontracts.org/docs/language-guide/upgrades.html#_preupgrade_and_postupgrade_system_methods)
 46 | 
 47 |       ```motoko
 48 |           system func preupgrade(){
 49 |               bucket_entries := bucket.preupgrade();
 50 |           };
 51 |       
 52 |           system func postupgrade(){
 53 |               bucket.postupgrade(bucket_entries);
 54 |               bucket_entries := [];
 55 |           };
 56 |       ```
 57 |       
 58 |   
 59 |   **nonupgradable** : This means that if you upgrade your canister, you will discard files stored in the stablememory
 60 |   ```motoko
 61 |   let bucket = Bucket.Bucket(false); // true : upgradable, false : nonupgradable
 62 |   ```
 63 | 
 64 | 
 65 | - Third,configure the **dfx.json**
 66 | 
 67 |   you should point out how much stable memory pages you want to use in dfx.json(recommendation : 131072)
 68 | 
 69 |   ```motoko
 70 |   "build" :{
 71 |      "args": "--max-stable-pages=131072" // the max size is 131072 [131072 = 8G / 64KB(each page size)]
 72 |   }
 73 |   ```
 74 | 
 75 | **[more details please read the demo](https://github.com/PrimLabs/Bucket/blob/main/src/Bucket/example.mo)**
 76 | 
 77 | ###  API
 78 | 
 79 | - **put** ：put the value into stablememory,use key to index
 80 | 
 81 |   ​            if you add it again with the same key, it will overwrite the previous file
 82 | 
 83 |   ```motoko
 84 |   public func put(key: Text, value : Blob): Result.Result<(), Error>
 85 |   ```
 86 | 
 87 |   tips: you can transform any type T to Text by using ``debug_show(t: T)``
 88 | 
 89 | - **append** ：put the value into stablememory,use key to index
 90 | 
 91 |   ​                    if added again with the same key, it will be merged with the previous file block
 92 | 
 93 |   ```motoko
 94 |   public func append(key: Text, value : Blob): Result.Result<(), Error>
 95 |   ```
 96 | 
 97 |   tips: you can transform any type T to Text by using ``debug_show(t: T)``
 98 | 
 99 | - **get** : use the key to get the value
100 | 
101 |   ```motoko
102 |   public func get(key: Text): Result.Result<[Blob], Error>
103 |   ```
104 | 
105 | - **preupgrade** : return entries
106 | 
107 |   ```motoko
108 |   public func preupgrade(): [(Text, [(Nat64, Nat)])]
109 |   ```
110 | 
111 | - **postupgrade**
112 | 
113 |   ```motoko
114 |   public func postupgrade(entries : [(Text, [(Nat64, Nat)])]): ()
115 |   ```
116 | 
117 | <span id="Bucket-HTTP"></span>
118 | 
119 | ##  Bucket-HTTP
120 | 
121 | The difference between Bucket-HTTP and Bucket is that Bucket-HTTP has built-in **http_request**, so people can query files through example : **canisterID.raw.ic0.app/static/key**
122 | 
123 | example
124 | 
125 | ```
126 | https://2fli5-jyaaa-aaaao-aabea-cai.raw.ic0.app/static/0
127 | ```
128 | 
129 | Due to the problem of IC mainnet, HTTP-StreamingCallback cannot work at present, so only files less than or equal to **3M** can be accessed through http.
130 | 
131 | **We will fix this deficiency as soon as possible.**
132 | 
133 | [**The preparation is almost the same as the above, just change Bucket to Bucket-HTTP**](#prework)
134 | 
135 | **[more details please read the demo](https://github.com/PrimLabs/Bucket/blob/main/src/Bucket-HTTP/example.mo)**
136 | 
137 | ###  API
138 | 
139 | - **put** ：put the value into stablememory,use key to index
140 | 
141 |   ​            if you add it again with the same key, it will overwrite the previous file
142 | 
143 |   ```motoko
144 |   public func put(key: Text, value : Blob): Result.Result<(), Error>
145 |   ```
146 | 
147 |   tips: you can transform any type T to Text by using ``debug_show(t: T)``
148 | 
149 | - **append** ：put the value into stablememory,use key to index
150 | 
151 |   ​                    if added again with the same key, it will be merged with the previous file block
152 | 
153 |   ```motoko
154 |   public func append(key: Text, value : Blob): Result.Result<(), Error>
155 |   ```
156 | 
157 |   tips: you can transform any type T to Text by using ``debug_show(t: T)``
158 | 
159 | - **get** : use the key to get the value
160 | 
161 |   ```motoko
162 |   public func get(key: Text): Result.Result<[Blob], Error>
163 |   ```
164 | 
165 | - **build_http** : Pass in the function that parses the key in the url,the key is used to get the value
166 | 
167 |   ATTENTION : YOU MUST SET YOUR DECODE FUNCITON OR REWRITE IT AND CALL THE BUILD FUNCTION TO ENABLE IT WHEN YOU NEED TO USE THE HTTP INTERFACE.
168 | 
169 |   ```motoko
170 |   public func build_http(fn_: DecodeUrl): ()
171 |   ```
172 | 
173 |   ```motoko
174 |   public type DecodeUrl = (Text) -> (Text);
175 |   ```
176 | 
177 | - **http_request**
178 | 
179 |   ```motoko
180 |   public func http_request(request: HttpRequest): HttpResponse
181 |   ```
182 | 
183 | - **preupgrade** : return entries
184 | 
185 |   ```motoko
186 |   public func preupgrade(): [(Text, [(Nat64, Nat)])]
187 |   ```
188 | 
189 | - **postupgrade**
190 | 
191 |   ```motoko
192 |   public func postupgrade(entries : [(Text, [(Nat64, Nat)])]): ()
193 |   ```
194 | 
195 | 
196 | ## Disclaimer
197 | 
198 | YOU EXPRESSLY ACKNOWLEDGE AND AGREE THAT USE OF THIS SOFTWARE IS AT YOUR SOLE RISK. AUTHORS OF THIS SOFTWARE SHALL NOT BE LIABLE FOR DAMAGES OF ANY TYPE, WHETHER DIRECT OR INDIRECT.
199 | 
200 | ## Contributing
201 | 
202 | <span id="hh"></span>
203 | 
204 | We'd like to collaborate with the community to provide better data storage standard implementation for the developers on the IC, if you have some ideas you'd like to discuss, submit an issue, if you want to improve the code or you made a different implementation, make a pull request!
205 | 


--------------------------------------------------------------------------------
/dfx.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "version": 1,
 3 |   "dfx": "0.9.3",
 4 |   "canisters": {
 5 |     "example": {
 6 |       "type": "motoko",
 7 |       "main": "src/Bucket-HTTP/example.mo"
 8 |     }
 9 |   },
10 |   "defaults": {
11 |     "build": {
12 |       "packtool": "",
13 |       "args": "--max-stable-pages=131072"
14 |     }
15 |   },
16 |   "networks": {
17 |     "local": {
18 |       "bind": "127.0.0.1:8000",
19 |       "type": "ephemeral"
20 |     }
21 |   }
22 | }
23 | 


--------------------------------------------------------------------------------
/src/Bucket-HTTP/Bucket-HTTP.mo:
--------------------------------------------------------------------------------
  1 | import TrieMap "mo:base/TrieMap";
  2 | import Result "mo:base/Result";
  3 | import Blob "mo:base/Blob";
  4 | import Array "mo:base/Array";
  5 | import Text "mo:base/Text";
  6 | import Nat64 "mo:base/Nat64";
  7 | import Nat "mo:base/Nat";
  8 | import SM "mo:base/ExperimentalStableMemory";
  9 | import Prim "mo:⛔";
 10 | 
 11 | module {
 12 | 
 13 |     public type Error = {
 14 |         #INSUFFICIENT_MEMORY;
 15 |         #BlobSizeError;
 16 |         #INVALID_KEY;
 17 |         #Append_Error;
 18 |     };
 19 | 
 20 |     public type HeaderField = (Text, Text);
 21 |     public type StreamingCallbackResponse = {
 22 |         body: Blob;
 23 |         token: ?CallbackToken;
 24 |     };
 25 |     public type CallbackToken = {
 26 |         index: Nat;
 27 |         key: Text;
 28 |     };
 29 |     public type StreamingCallback = query (CallbackToken) -> async (StreamingCallbackResponse);
 30 |     public type StreamingStrategy = {
 31 |         #Callback: {
 32 |             callback: StreamingCallback;
 33 |             token: CallbackToken;
 34 |         }
 35 |     };
 36 |     public type HttpRequest = {
 37 |         method: Text;
 38 |         url: Text;
 39 |         headers: [HeaderField];
 40 |         body: Blob;
 41 |     };
 42 |     public type HttpResponse = {
 43 |         status_code: Nat16;
 44 |         headers: [HeaderField];
 45 |         body: Blob;
 46 |         streaming_strategy: ?StreamingStrategy;
 47 |     };   
 48 |     
 49 |     public type DecodeUrl = (Text) -> (Text, Text);
 50 |     
 51 |     public class BucketHttp(upgradable : Bool) {
 52 |         private let THRESHOLD               = 6442450944;
 53 |         // MAX_PAGE_SIZE = 8 GB(total size of stable memory currently) / 64 KB(each page size = 64 KB)
 54 |         private let MAX_PAGE_BYTE           = 65536;
 55 |         private let MAX_PAGE_NUMBER         = 131072 : Nat64;
 56 |         private let MAX_QUERY_SIZE          = 3144728;
 57 |         private var offset                  = 8; // 0 - 7 is used for offset
 58 |         private var decodeurl: ?DecodeUrl   = null;
 59 |         var assets = TrieMap.TrieMap<Text, [(Nat64, Nat)]>(Text.equal, Text.hash);
 60 | 
 61 |         public func put(key: Text, value : Blob): Result.Result<(), Error> {
 62 |             switch(_getField(value.size())) {
 63 |                 case(#ok(field)) {
 64 |                     assets.put(key, [field]);
 65 |                     _storageData(field.0, value);
 66 |                 };
 67 |                 case(#err(err)) { return #err(err) };
 68 |             };
 69 |             #ok(())
 70 |         };
 71 | 
 72 |         public func append(key: Text, value : Blob): Result.Result<(), Error> {
 73 |             switch(_getField(value.size())) {
 74 |                 case(#ok(field)) {
 75 |                     switch(assets.get(key)){
 76 |                         case null { return #err(#Append_Error);};
 77 |                         case(?pre_field){
 78 |                             let present_field = Array.append<(Nat64, Nat)>(pre_field, [field]);
 79 |                             assets.put(key, present_field);
 80 |                         };
 81 |                     };
 82 |                     _storageData(field.0, value);
 83 |                 };
 84 |                 case(#err(err)) { return #err(err) };
 85 |             };
 86 |             #ok(())
 87 |         };
 88 | 
 89 |         public func get(key: Text): Result.Result<[Blob], Error> {
 90 |             switch(assets.get(key)) {
 91 |                 case(null) { return #err(#INVALID_KEY) };
 92 |                 case(?field) {
 93 |                     let res = Array.init<Blob>(field.size(), "":Blob);
 94 |                     var index = 0;
 95 |                     for(f in field.vals()){
 96 |                         res[index] := _loadFromSM(f);
 97 |                         index += 1;
 98 |                     };
 99 |                     #ok(Array.freeze<Blob>(res))
100 |                 };
101 |             };
102 |         };
103 | 
104 |         public func http_request(request: HttpRequest,callbackfunc: StreamingCallback): HttpResponse {
105 |             switch(decodeurl) {
106 |                 case(null) { return errStaticpage("Decodeurl Funtion Wrong");};
107 |                 case(?_decodeurl) {
108 |                     let info = _decodeurl(request.url);
109 |                     let fileKey = info.1;let fileType = info.0;
110 |                     switch(get(fileKey)) {
111 |                         case(#err(err)) { return errStaticpage("get wrong");};
112 |                         case(#ok(payload)) {
113 |                             let number = payload.size();
114 |                             if(number == 1) {
115 |                                 return {
116 |                                     status_code = 200;
117 |                                     headers = getContentType(fileType);
118 |                                     body = payload[0];
119 |                                     streaming_strategy = null;
120 |                                 };
121 |                             } else {
122 |                                 return {
123 |                                     status_code = 200;
124 |                                     headers = getContentType(fileType);
125 |                                     body = payload[0];
126 |                                     streaming_strategy = ?#Callback({
127 |                                         callback = callbackfunc;
128 |                                         token = {
129 |                                             index = 1;
130 |                                             key = fileKey;
131 |                                         };
132 |                                     });
133 |                                 }
134 |                             }
135 |                         };
136 |                     };                    
137 |                 };
138 |             };
139 |             errStaticpage("Somting Wrong")
140 |         };
141 | 
142 |         public func streamingCallback(token: CallbackToken): StreamingCallbackResponse {
143 |             var payload: [Blob] = [];
144 |             switch(get(token.key)) {
145 |                 case(#err(err)) {};
146 |                 case(#ok(ans)) { payload := ans;};
147 |             };
148 |             {
149 |                 body  = payload[token.index];
150 |                 token = if(token.index + 1 == payload.size() ) {
151 |                     null
152 |                 } else {
153 |                     ?{
154 |                         index = token.index + 1;
155 |                         key   = token.key;
156 |                     }
157 |                 };
158 |             }
159 |         };
160 | 
161 |         public func build_http(fn_: DecodeUrl): () {
162 |             decodeurl := ?fn_;
163 |         };
164 |         
165 |         // return entries
166 |         public func preupgrade(): [(Text, [(Nat64, Nat)])] {
167 |             SM.storeNat64(0 : Nat64, Nat64.fromNat(offset));
168 |             var index = 0;
169 |             var assets_entries = Array.init<(Text, [(Nat64, Nat)])>(assets.size(), ("", []));
170 |             for (kv in assets.entries()) {
171 |                 assets_entries[index] := kv;
172 |                 index += 1;
173 |             };
174 |             Array.freeze<(Text, [(Nat64, Nat)])>(assets_entries)
175 |         };
176 | 
177 |         public func postupgrade(entries : [(Text, [(Nat64, Nat)])]): () {
178 |             offset := Nat64.toNat(SM.loadNat64(0:Nat64));
179 |             assets := TrieMap.fromEntries<Text, [(Nat64, Nat)]>(entries.vals(), Text.equal, Text.hash);
180 |         };
181 | 
182 |         private func getContentType(fileType: Text): [HeaderField] {
183 |             if(fileType == "gif") return [("Content-Type", "image/gif")];
184 |             if(fileType == "jpeg") return [("Content-Type", "image/jpeg")];
185 |             if(fileType == "png") return [("Content-Type", "image/png")];
186 |             if(fileType == "pdf") return [("Content-Type", "application/pdf")];
187 |             if(fileType == "doc") return [("Content-Type", "application/msword")];
188 |             if(fileType == "mp3") return [("Content-Type", "audio/mp3")];
189 |             if(fileType == "mp4") return [("Content-Type", "video/mp4")];
190 |             if(fileType == "txt") return [("Content-Type", "text/plain")];
191 |             if(fileType == "ppt") return [("Content-Type", "application/vnd.ms-powerpoint")];
192 |             if(fileType == "css") return [("Content-Type", "text/css")];
193 |             return [("Content-Type", "text/html; charset=utf-8")];
194 |         };
195 | 
196 |         private func _loadFromSM(field : (Nat64, Nat)) : Blob {
197 |             SM.loadBlob(field.0, field.1)
198 |         };
199 | 
200 |         private func _getField(total_size : Nat) : Result.Result<(Nat64, Nat), Error> {
201 |             switch (_inspectSize(total_size)) {
202 |                 case (#err(err)) { #err(err) };
203 |                 case (#ok(_)) {
204 |                     let field = (Nat64.fromNat(offset), total_size);
205 |                     _growStableMemoryPage(total_size);
206 |                     offset += total_size;
207 |                     #ok(field)
208 |                 };
209 |             }
210 |         };
211 | 
212 |         // check total_size
213 |         private func _inspectSize(total_size : Nat) : Result.Result<(), Error> {
214 |             if (total_size <= _getAvailableMemorySize()) { #ok(()) } else { #err(#INSUFFICIENT_MEMORY) };
215 |         };
216 | 
217 |         // upload时根据分配好的write_page以vals的形式写入数据
218 |         // When uploading, write data in the form of vals according to the assigned write_page
219 |         private func _storageData(start : Nat64, data : Blob) {
220 |             SM.storeBlob(start, data)
221 |         };
222 | 
223 |         // return available memory size can be allocated
224 |         private func _getAvailableMemorySize() : Nat{
225 |             if(upgradable){
226 |                 assert(THRESHOLD >= Prim.rts_memory_size() + offset);
227 |                 THRESHOLD - Prim.rts_memory_size() - offset
228 |             }else{
229 |                 THRESHOLD - offset
230 |             }
231 |         };
232 | 
233 |         // grow SM memory pages of size "size"
234 |         private func _growStableMemoryPage(size : Nat) {
235 |             if(offset == 8){ ignore SM.grow(1 : Nat64) };
236 |             let available_mem : Nat = Nat64.toNat(SM.size()) * MAX_PAGE_BYTE + 1 - offset;
237 |             if (available_mem < size) {
238 |                 let need_allo_size : Nat = size - available_mem;
239 |                 let growPage = Nat64.fromNat(need_allo_size / MAX_PAGE_BYTE + 1);
240 |                 ignore SM.grow(growPage);
241 |             }
242 |         };
243 | 
244 |         private func errStaticpage(err: Text): HttpResponse {
245 |             {
246 |                 status_code = 404;
247 |                 headers = [("Content-Type", "text/plain")];
248 |                 body = Text.encodeUtf8(err);
249 |                 streaming_strategy = null;
250 |             }
251 |         };
252 | 
253 |     };
254 | };
255 | 


--------------------------------------------------------------------------------
/src/Bucket-HTTP/example.mo:
--------------------------------------------------------------------------------
 1 | import BucketHttp "Bucket-HTTP";
 2 | import Blob "mo:base/Blob";
 3 | import Text "mo:base/Text";
 4 | import Array "mo:base/Array";
 5 | import Result "mo:base/Result";
 6 | import Iter "mo:base/Iter";
 7 | import Nat "mo:base/Nat";
 8 | import Debug "mo:base/Debug";
 9 | import SM "mo:base/ExperimentalStableMemory";
10 | 
11 | actor example{   
12 |     type HttpRequest = BucketHttp.HttpRequest;
13 |     type HttpResponse = BucketHttp.HttpResponse;
14 |     type CallbackToken = BucketHttp.CallbackToken;
15 |     type StreamingCallbackResponse = BucketHttp.StreamingCallbackResponse;
16 |     type Error = BucketHttp.Error;
17 |     
18 |     stable var entries : [(Text, [(Nat64, Nat)])] = [];
19 |     let bucket = BucketHttp.BucketHttp(true); // true : upgradable, false : unupgradable
20 |     
21 |     // CanisterId.raw.ic0.app/fileType/fileKey 
22 |     private func decodeurl(url: Text): (Text, Text) { //(fileType, fileKey)
23 |         let path = Iter.toArray(Text.tokens(url, #text("/")));
24 |         if(path.size() == 2) return (path[0], path[1]);
25 |         return ("txt", "Wrong key");
26 |     };
27 | 
28 |     public shared func build_http(): async () {
29 |         bucket.build_http(decodeurl);
30 |     };
31 | 
32 |     public query func streamingCallback(token: CallbackToken): async StreamingCallbackResponse {
33 |         bucket.streamingCallback(token)
34 |     };
35 | 
36 |     public query func http_request(request: HttpRequest): async HttpResponse {
37 |         bucket.http_request(request, streamingCallback)
38 |     };
39 | 
40 |     public query func getBlob(key: Text) : async Result.Result<[Blob], Error>{
41 |         switch(bucket.get(key)){
42 |             case(#err(e)){ #err(e) };
43 |             case(#ok(blob)){
44 |                 #ok(blob)
45 |             }
46 |         }
47 |     };
48 | 
49 |     public shared func putImg(key: Text,value: Blob,index: Nat) : async Result.Result<(), Error>{
50 |         if(index == 1) {
51 |             switch(bucket.put(key, value)){
52 |                 case(#err(e)){ return #err(e) };
53 |                 case(_){ return #ok(());};
54 |             };
55 |         };
56 |         if(index > 1) {
57 |             switch(bucket.append(key, value)){
58 |                 case(#err(e)){ return #err(e) };
59 |                 case(_){ return #ok(());};
60 |             };
61 |         };
62 |         #ok(())
63 |     };
64 | 
65 |     public shared func putBlob() : async Result.Result<(), Error>{
66 |         let key = "key";
67 |         let value = Text.encodeUtf8("this is the value");
68 |         switch(bucket.put(key, value)){
69 |             case(#err(e)){ return #err(e) };
70 |             case(_){};
71 |         };
72 |         #ok(())
73 |     };
74 | 
75 |     system func preupgrade(){
76 |         entries := bucket.preupgrade();
77 |     };
78 | 
79 |     system func postupgrade(){
80 |         bucket.postupgrade(entries);
81 |         entries := [];
82 |     };
83 | 
84 | }
85 | 


--------------------------------------------------------------------------------
/src/Bucket/Bucket.mo:
--------------------------------------------------------------------------------
  1 | import TrieMap "mo:base/TrieMap";
  2 | import Result "mo:base/Result";
  3 | import Blob "mo:base/Blob";
  4 | import Array "mo:base/Array";
  5 | import Text "mo:base/Text";
  6 | import Nat64 "mo:base/Nat64";
  7 | import SM "mo:base/ExperimentalStableMemory";
  8 | import Prim "mo:⛔";
  9 | 
 10 | module {
 11 | 
 12 |     public type Error = {
 13 |         #INSUFFICIENT_MEMORY;
 14 |         #BlobSizeError;
 15 |         #INVALID_KEY;
 16 |         #Append_Error;
 17 |     };
 18 | 
 19 |     public class Bucket(upgradable : Bool) {
 20 |         private let THRESHOLD               = 6442450944;
 21 |         // MAX_PAGE_SIZE = 8 GB(total size of stable memory currently) / 64 KB(each page size = 64 KB)
 22 |         private let MAX_PAGE_BYTE               = 65536;
 23 |         private let MAX_PAGE_NUMBER         = 131072 : Nat64;
 24 |         private let MAX_QUERY_SIZE          = 3144728;
 25 |         private var offset                  = 8; // 0 - 7 is used for offset
 26 |         var assets = TrieMap.TrieMap<Text, [(Nat64, Nat)]>(Text.equal, Text.hash);
 27 | 
 28 |         public func put(key: Text, value : Blob): Result.Result<(), Error> {
 29 |             switch(_getField(value.size())) {
 30 |                 case(#ok(field)) {
 31 |                     assets.put(key, [field]);
 32 |                     _storageData(field.0, value);
 33 |                 };
 34 |                 case(#err(err)) { return #err(err) };
 35 |             };
 36 |             #ok(())
 37 |         };
 38 | 
 39 |         public func append(key: Text, value : Blob): Result.Result<(), Error> {
 40 |             switch(_getField(value.size())) {
 41 |                 case(#ok(field)) {
 42 |                     switch(assets.get(key)){
 43 |                         case null { return #err(#Append_Error);};
 44 |                         case(?pre_field){
 45 |                             let present_field = Array.append<(Nat64, Nat)>(pre_field, [field]);
 46 |                             assets.put(key, present_field);
 47 |                         };
 48 |                     };
 49 |                     _storageData(field.0, value);
 50 |                 };
 51 |                 case(#err(err)) { return #err(err) };
 52 |             };
 53 |             #ok(())
 54 |         };
 55 |         
 56 |         public func get(key: Text): Result.Result<[Blob], Error> {
 57 |             switch(assets.get(key)) {
 58 |                 case(null) { return #err(#INVALID_KEY) };
 59 |                 case(?field) {
 60 |                     let res = Array.init<Blob>(field.size(), "":Blob);
 61 |                     var index = 0;
 62 |                     for(f in field.vals()){
 63 |                         res[index] := _loadFromSM(f);
 64 |                         index += 1;
 65 |                     };
 66 |                     #ok(Array.freeze<Blob>(res))
 67 |                 };
 68 |             };
 69 |         };
 70 | 
 71 |         // return entries
 72 |         public func preupgrade(): [(Text, [(Nat64, Nat)])] {
 73 |             SM.storeNat64(0 : Nat64, Nat64.fromNat(offset));
 74 |             var index = 0;
 75 |             var assets_entries = Array.init<(Text, [(Nat64, Nat)])>(assets.size(), ("", []));
 76 |             for (kv in assets.entries()) {
 77 |                 assets_entries[index] := kv;
 78 |                 index += 1;
 79 |             };
 80 |             Array.freeze<(Text, [(Nat64, Nat)])>(assets_entries)
 81 |         };
 82 | 
 83 |         public func postupgrade(entries : [(Text, [(Nat64, Nat)])]): () {
 84 |             offset := Nat64.toNat(SM.loadNat64(0:Nat64));
 85 |             assets := TrieMap.fromEntries<Text, [(Nat64, Nat)]>(entries.vals(), Text.equal, Text.hash);
 86 |         };
 87 | 
 88 |         private func _loadFromSM(field : (Nat64, Nat)) : Blob {
 89 |             SM.loadBlob(field.0, field.1)
 90 |         };
 91 | 
 92 |         private func _getField(total_size : Nat) : Result.Result<(Nat64, Nat), Error> {
 93 |             switch (_inspectSize(total_size)) {
 94 |                 case (#err(err)) { #err(err) };
 95 |                 case (#ok(_)) {
 96 |                     let field = (Nat64.fromNat(offset), total_size);
 97 |                     _growStableMemoryPage(total_size);
 98 |                     offset += total_size;
 99 |                     #ok(field)
100 |                 };
101 |             }
102 |         };
103 | 
104 |         // check total_size
105 |         private func _inspectSize(total_size : Nat) : Result.Result<(), Error> {
106 |             if (total_size <= _getAvailableMemorySize()) { #ok(()) } else { #err(#INSUFFICIENT_MEMORY) };
107 |         };
108 | 
109 |         // upload时根据分配好的write_page以vals的形式写入数据
110 |         // When uploading, write data in the form of vals according to the assigned write_page
111 |         private func _storageData(start : Nat64, data : Blob) {
112 |             SM.storeBlob(start, data)
113 |         };
114 | 
115 |         // return available memory size can be allocated
116 |         private func _getAvailableMemorySize() : Nat{
117 |             if(upgradable){
118 |                 assert(THRESHOLD >= Prim.rts_memory_size() + offset);
119 |                 THRESHOLD - Prim.rts_memory_size() - offset
120 |             }else{
121 |                 THRESHOLD - offset
122 |             }
123 |         };
124 | 
125 |         // grow SM memory pages of size "size"
126 |         private func _growStableMemoryPage(size : Nat) {
127 |             if(offset == 8){ ignore SM.grow(1 : Nat64) };
128 |             let available_mem : Nat = Nat64.toNat(SM.size()) * MAX_PAGE_BYTE + 1 - offset;
129 |             if (available_mem < size) {
130 |                 let need_allo_size : Nat = size - available_mem;
131 |                 let growPage = Nat64.fromNat(need_allo_size / MAX_PAGE_BYTE + 1);
132 |                 ignore SM.grow(growPage);
133 |             }
134 |         };
135 | 
136 |     };
137 | };
138 | 


--------------------------------------------------------------------------------
/src/Bucket/example.mo:
--------------------------------------------------------------------------------
  1 | import Bucket "Bucket";
  2 | import Blob "mo:base/Blob";
  3 | import Text "mo:base/Text";
  4 | import Array "mo:base/Array";
  5 | import Result "mo:base/Result";
  6 | import Nat "mo:base/Nat";
  7 | import Debug "mo:base/Debug";
  8 | 
  9 | actor example{
 10 | 
 11 |     type Error = Bucket.Error;
 12 |     type S = {
 13 |         text : Text;
 14 |         bool : Bool
 15 |     };
 16 |     stable var bucket_entries : [(Text, [(Nat64, Nat)])] = [];
 17 |     let bucket = Bucket.Bucket(true); // true : upgradable, false : unupgradable
 18 | 
 19 |     public query func getBlob(key : Text) : async Result.Result<[Blob], Error>{
 20 |         switch(bucket.get(key)){
 21 |             case(#err(e)){ #err(e) };
 22 |             case(#ok(blob)){
 23 |                 #ok(blob)
 24 |             }
 25 |         }
 26 |     };
 27 | 
 28 |     public query func get(key : Text) : async Result.Result<[S], Error>{
 29 |         switch(bucket.get(key)){
 30 |             case(#err(info)){ #err(info) };
 31 |             case(#ok(data)){ #ok(deserialize(data)) };
 32 |         };
 33 |     };
 34 | 
 35 |     public func put() : async Result.Result<(), Error>{
 36 |         let key = "key";
 37 |         let value_1 : S = {
 38 |             text = "this is the first slice of value";
 39 |             bool = true
 40 |         };
 41 |         let value_2 : S = {
 42 |             text = "this is the second slice of value";
 43 |             bool = false
 44 |         };
 45 |         switch(bucket.put(key, serialize(value_1))){
 46 |             case(#err(e)){ return #err(e) };
 47 |             case(_){};
 48 |         };
 49 |         // you can storage the two different value using the same key
 50 |         switch(bucket.append(key, serialize(value_2))){
 51 |             case(#err(e)){ return #err(e) };
 52 |             case(_){};
 53 |         };
 54 |         #ok(())
 55 |     };
 56 | 
 57 |     public func putBlob() : async Result.Result<(), Error>{
 58 |         let key = "key";
 59 |         let value = Text.encodeUtf8("this is the value");
 60 |         switch(bucket.put(key, value)){
 61 |             case(#err(e)){ return #err(e) };
 62 |             case(_){};
 63 |         };
 64 |         #ok(())
 65 |     };
 66 | 
 67 |     system func preupgrade(){
 68 |         bucket_entries := bucket.preupgrade();
 69 |     };
 70 | 
 71 |     system func postupgrade(){
 72 |         bucket.postupgrade(bucket_entries);
 73 |         bucket_entries := [];
 74 |     };
 75 | 
 76 |     // you should encode the segment of the struct into nat8
 77 |     // then you should merge them and transform the [Nat8] to Blob
 78 |     private func serialize(s : S) : Blob{
 79 |         let bool_nat8 = if(s.bool){
 80 |             1 : Nat8
 81 |         }else{ 0 : Nat8 };
 82 |         let text_blob = Text.encodeUtf8(s.text);
 83 |         let text_nat8 = Blob.toArray(text_blob);
 84 |         let serialize_data = Array.append<Nat8>(text_nat8, [bool_nat8]);
 85 |         Blob.fromArray(serialize_data)
 86 |     };
 87 | 
 88 |     private func deserialize(data : [Blob]) : [S] {
 89 |         let res = Array.init<S>(data.size(), {
 90 |             text = "";
 91 |             bool = true;
 92 |         });
 93 |         var res_index = 0;
 94 |         for(d in data.vals()){
 95 |             let raw = Blob.toArray(d);
 96 |             let bool = if(raw[Nat.sub(raw.size(), 1)] == 1){ true }else{ false };
 97 |             let text = Array.init<Nat8>(Nat.sub(data.size(), 2), 0:Nat8);// the last byte is used to store the "bool"
 98 |             var index = 0;
 99 |             label l for(d in raw.vals()){
100 |                 text[index] := d;
101 |                 index += 1;
102 |                 if(index == text.size()){ break l };
103 |             };
104 |             let t =
105 |                 switch(Text.decodeUtf8(Blob.fromArray(Array.freeze<Nat8>(text)))){
106 |                     case null { "" };
107 |                     case(?te){ te };
108 |                 };
109 |             res[res_index] :=
110 |                 {
111 |                     text = t;
112 |                     bool = bool
113 |                 };
114 |             res_index += 1;
115 |         };
116 |         Array.freeze(res)
117 |     };
118 | 
119 | }
120 | 


--------------------------------------------------------------------------------