├── .travis.yml ├── README.md ├── src ├── main │ └── java │ │ └── com │ │ └── scoopit │ │ └── weedfs │ │ └── client │ │ ├── status │ │ ├── AbstractNode.java │ │ ├── MasterStatus.java │ │ ├── Rack.java │ │ ├── DataCenter.java │ │ ├── VolumeStatus.java │ │ ├── DataNode.java │ │ ├── Layout.java │ │ ├── Volume.java │ │ └── Topology.java │ │ ├── caching │ │ ├── LookupCache.java │ │ └── MapLookupCache.java │ │ ├── net │ │ ├── Result.java │ │ ├── WriteResult.java │ │ ├── LookupResult.java │ │ └── AssignResult.java │ │ ├── Location.java │ │ ├── WeedFSFileNotFoundException.java │ │ ├── WeedFSException.java │ │ ├── Assignation.java │ │ ├── WeedFSFile.java │ │ ├── AssignParams.java │ │ ├── ReplicationStrategy.java │ │ ├── WeedFSClientBuilder.java │ │ ├── WeedFSClientMock.java │ │ ├── WeedFSClient.java │ │ └── WeedFSClientImpl.java └── test │ └── java │ └── com │ └── scoopit │ └── weedfs │ └── benchmark │ ├── LimitTest.java │ ├── Maestro.java │ ├── StatsCollector.java │ ├── LoadTest.java │ ├── ReadAndCheckFile.java │ └── UploadRandomFile.java ├── .project ├── .gitignore └── pom.xml /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - openjdk7 4 | notifications: 5 | email: 6 | - nicolas.lalevee@hibnet.org -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Weed-FS-Java-Client 2 | =================== 3 | 4 | Low level java client library for the [Weed-FS](https://code.google.com/p/weed-fs/) REST interface. 5 | 6 | Feel free to contribute. 7 | 8 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/status/AbstractNode.java: -------------------------------------------------------------------------------- 1 | package com.scoopit.weedfs.client.status; 2 | 3 | public abstract class AbstractNode { 4 | public int Free; 5 | public int Max; 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/status/MasterStatus.java: -------------------------------------------------------------------------------- 1 | package com.scoopit.weedfs.client.status; 2 | 3 | public class MasterStatus { 4 | public Topology Topology; 5 | public String Version; 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/status/Rack.java: -------------------------------------------------------------------------------- 1 | package com.scoopit.weedfs.client.status; 2 | 3 | import java.util.List; 4 | 5 | public class Rack extends AbstractNode { 6 | public List DataNodes; 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/status/DataCenter.java: -------------------------------------------------------------------------------- 1 | package com.scoopit.weedfs.client.status; 2 | 3 | import java.util.List; 4 | 5 | public class DataCenter extends AbstractNode { 6 | public List Racks; 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/status/VolumeStatus.java: -------------------------------------------------------------------------------- 1 | package com.scoopit.weedfs.client.status; 2 | 3 | import java.util.List; 4 | 5 | public class VolumeStatus { 6 | public String Version; 7 | public List Volumes; 8 | } 9 | -------------------------------------------------------------------------------- /src/test/java/com/scoopit/weedfs/benchmark/LimitTest.java: -------------------------------------------------------------------------------- 1 | package com.scoopit.weedfs.benchmark; 2 | 3 | import com.scoopit.weedfs.client.WeedFSClient; 4 | import com.scoopit.weedfs.client.WeedFSClientBuilder; 5 | 6 | public class LimitTest { 7 | 8 | public static void main(String[] args) { 9 | 10 | WeedFSClient client = WeedFSClientBuilder.createBuilder().build(); 11 | 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/caching/LookupCache.java: -------------------------------------------------------------------------------- 1 | package com.scoopit.weedfs.client.caching; 2 | 3 | import java.util.List; 4 | 5 | import com.scoopit.weedfs.client.Location; 6 | 7 | public interface LookupCache { 8 | 9 | List lookup(long volumeId); 10 | 11 | void invalidate(long volumeId); 12 | 13 | void invalidate(); 14 | 15 | void setLocation(long volumeId, List locations); 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/status/DataNode.java: -------------------------------------------------------------------------------- 1 | package com.scoopit.weedfs.client.status; 2 | 3 | import com.scoopit.weedfs.client.Location; 4 | 5 | public class DataNode extends AbstractNode { 6 | public String PublicUrl; 7 | public String Url; 8 | public int Volumes; 9 | 10 | public Location asLocation() { 11 | Location ret = new Location(); 12 | ret.publicUrl = PublicUrl; 13 | ret.url = Url; 14 | return ret; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/status/Layout.java: -------------------------------------------------------------------------------- 1 | package com.scoopit.weedfs.client.status; 2 | 3 | import java.util.List; 4 | 5 | import com.scoopit.weedfs.client.ReplicationStrategy; 6 | 7 | public class Layout { 8 | public String collection; 9 | public String replication; 10 | public List writables; 11 | 12 | public ReplicationStrategy getReplicationStrategy() { 13 | return ReplicationStrategy.fromParameterValue(replication); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | weed-fs-client 4 | NO_M2ECLIPSE_SUPPORT: Project files created with the maven-eclipse-plugin are not supported in M2Eclipse. 5 | 6 | 7 | 8 | org.eclipse.jdt.core.javabuilder 9 | 10 | 11 | 12 | org.eclipse.jdt.core.javanature 13 | 14 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/status/Volume.java: -------------------------------------------------------------------------------- 1 | package com.scoopit.weedfs.client.status; 2 | 3 | import com.scoopit.weedfs.client.ReplicationStrategy; 4 | 5 | public class Volume { 6 | public int Id; 7 | public long Size; 8 | public String RepType; 9 | public String Collection; 10 | public String Version; 11 | public long FileCount; 12 | public long DeleteCount; 13 | public long DeletedByteCount; 14 | public boolean ReadOnly; 15 | 16 | public ReplicationStrategy getReplicationStrategy() { 17 | return ReplicationStrategy.fromParameterValue(RepType); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by http://gitignore.io 2 | 3 | ### Maven ### 4 | target/ 5 | 6 | ### Java ### 7 | *.class 8 | 9 | # Package Files # 10 | *.jar 11 | *.war 12 | *.ear 13 | 14 | ### Eclipse ### 15 | *.pydevproject 16 | .project 17 | .metadata 18 | bin/** 19 | tmp/** 20 | tmp/**/* 21 | *.tmp 22 | *.bak 23 | *.swp 24 | *~.nib 25 | local.properties 26 | .classpath 27 | .settings/ 28 | .loadpath 29 | 30 | # External tool builders 31 | .externalToolBuilders/ 32 | 33 | # Locally stored "Eclipse launch configurations" 34 | *.launch 35 | 36 | # CDT-specific 37 | .cproject 38 | 39 | # PDT-specific 40 | .buildpath 41 | 42 | # Mac OS 43 | .DS_Store 44 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/caching/MapLookupCache.java: -------------------------------------------------------------------------------- 1 | package com.scoopit.weedfs.client.caching; 2 | 3 | import java.util.List; 4 | import java.util.concurrent.ConcurrentHashMap; 5 | 6 | import com.scoopit.weedfs.client.Location; 7 | 8 | public class MapLookupCache implements LookupCache { 9 | ConcurrentHashMap> cache = new ConcurrentHashMap<>(); 10 | 11 | @Override 12 | public void invalidate() { 13 | cache = new ConcurrentHashMap<>(); 14 | } 15 | 16 | @Override 17 | public void invalidate(long volumeId) { 18 | cache.remove(volumeId); 19 | } 20 | 21 | @Override 22 | public List lookup(long volumeId) { 23 | return cache.get(volumeId); 24 | } 25 | 26 | @Override 27 | public void setLocation(long volumeId, List locations) { 28 | cache.put(volumeId, locations); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/net/Result.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2013 Scoop IT SAS (http://scoop.it/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Philippe GASSMANN 18 | * Jean-Baptiste BELLET 19 | */ 20 | package com.scoopit.weedfs.client.net; 21 | 22 | public abstract class Result { 23 | public String error; 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/net/WriteResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2013 Scoop IT SAS (http://scoop.it/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Philippe GASSMANN 18 | * Jean-Baptiste BELLET 19 | */ 20 | package com.scoopit.weedfs.client.net; 21 | 22 | public class WriteResult extends Result { 23 | public int size; 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/net/LookupResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2013 Scoop IT SAS (http://scoop.it/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Philippe GASSMANN 18 | * Jean-Baptiste BELLET 19 | */ 20 | package com.scoopit.weedfs.client.net; 21 | 22 | import java.util.List; 23 | 24 | import com.scoopit.weedfs.client.Location; 25 | 26 | public class LookupResult extends Result { 27 | 28 | public List locations; 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/Location.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2013 Scoop IT SAS (http://scoop.it/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Philippe GASSMANN 18 | * Jean-Baptiste BELLET 19 | */ 20 | package com.scoopit.weedfs.client; 21 | 22 | public class Location { 23 | public String publicUrl; 24 | public String url; 25 | 26 | @Override 27 | public String toString() { 28 | return "Location [publicUrl=" + publicUrl + ", url=" + url + "]"; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/WeedFSFileNotFoundException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2013 Scoop IT SAS (http://scoop.it/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Philippe GASSMANN 18 | * Jean-Baptiste BELLET 19 | */ 20 | package com.scoopit.weedfs.client; 21 | 22 | public class WeedFSFileNotFoundException extends WeedFSException { 23 | 24 | private static final long serialVersionUID = 1L; 25 | 26 | public WeedFSFileNotFoundException(WeedFSFile file, Location location) { 27 | super(file.fid + " not found on " + location.publicUrl); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/WeedFSException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2013 Scoop IT SAS (http://scoop.it/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Philippe GASSMANN 18 | * Jean-Baptiste BELLET 19 | */ 20 | package com.scoopit.weedfs.client; 21 | 22 | import java.io.IOException; 23 | 24 | public class WeedFSException extends IOException { 25 | 26 | private static final long serialVersionUID = 1L; 27 | 28 | public WeedFSException(String reason) { 29 | super(reason); 30 | } 31 | 32 | public WeedFSException(Exception cause) { 33 | super(cause); 34 | } 35 | 36 | public WeedFSException(String reason, Exception cause) { 37 | super(reason, cause); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/com/scoopit/weedfs/benchmark/Maestro.java: -------------------------------------------------------------------------------- 1 | package com.scoopit.weedfs.benchmark; 2 | 3 | import java.util.Random; 4 | 5 | import akka.actor.ActorRef; 6 | import akka.actor.Props; 7 | import akka.actor.UntypedActor; 8 | 9 | public class Maestro extends UntypedActor { 10 | 11 | Random random = new Random(); 12 | 13 | final ActorRef uploadActor; 14 | final ActorRef readActor; 15 | 16 | public static Props mkProps(ActorRef uploadActor, ActorRef readActor) { 17 | return Props.create(Maestro.class, uploadActor, readActor); 18 | } 19 | 20 | public Maestro(ActorRef uploadActor, ActorRef readActor) { 21 | this.uploadActor = uploadActor; 22 | this.readActor = readActor; 23 | } 24 | 25 | @Override 26 | public void onReceive(Object arg0) throws Exception { 27 | 28 | // write or read ? 29 | 30 | boolean write = LoadTest.fsDb.size() == 0 || random.nextInt(100) > 85; 31 | 32 | if (write) { 33 | uploadActor.tell(new Object(), getSelf()); 34 | } else { 35 | // find a file on the db... 36 | int n = random.nextInt(LoadTest.fsDb.size()), i = 0; 37 | for (String fid : LoadTest.fsDb.keySet()) { 38 | if (++i == n) { 39 | readActor.tell(fid, getSelf()); 40 | } 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/net/AssignResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2013 Scoop IT SAS (http://scoop.it/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Philippe GASSMANN 18 | * Jean-Baptiste BELLET 19 | */ 20 | package com.scoopit.weedfs.client.net; 21 | 22 | import com.scoopit.weedfs.client.Location; 23 | import com.scoopit.weedfs.client.WeedFSFile; 24 | 25 | public class AssignResult extends Result { 26 | public int count; 27 | public String fid; 28 | public String publicUrl; 29 | public String url; 30 | 31 | public Location getLocation() { 32 | Location ret = new Location(); 33 | ret.publicUrl = publicUrl; 34 | ret.url = url; 35 | return ret; 36 | } 37 | 38 | public WeedFSFile getWeedFSFile() { 39 | return new WeedFSFile(fid); 40 | } 41 | 42 | public int getCount() { 43 | return count; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/Assignation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2013 Scoop IT SAS (http://scoop.it/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Philippe GASSMANN 18 | * Jean-Baptiste BELLET 19 | */ 20 | package com.scoopit.weedfs.client; 21 | 22 | import com.scoopit.weedfs.client.net.AssignResult; 23 | 24 | public class Assignation { 25 | 26 | public WeedFSFile weedFSFile; 27 | 28 | public Location location; 29 | 30 | int versionCount; 31 | 32 | public Assignation(AssignResult result) { 33 | this.weedFSFile = result.getWeedFSFile(); 34 | this.location = result.getLocation(); 35 | this.versionCount = result.getCount(); 36 | } 37 | 38 | public Assignation() { 39 | } 40 | 41 | @Override 42 | public String toString() { 43 | return "AssignedWeedFSFile [weedFSFile=" + weedFSFile + ", location=" + location + ", versionCount=" + versionCount + "]"; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/WeedFSFile.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2013 Scoop IT SAS (http://scoop.it/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Philippe GASSMANN 18 | * Jean-Baptiste BELLET 19 | */ 20 | package com.scoopit.weedfs.client; 21 | 22 | public class WeedFSFile { 23 | 24 | public final String fid; 25 | 26 | public int version = 0; 27 | 28 | public WeedFSFile(String fid) { 29 | this.fid = fid; 30 | } 31 | 32 | public long getVolumeId() { 33 | int pos = fid.indexOf(','); 34 | if (pos == -1) { 35 | throw new IllegalArgumentException("Cannot parse fid: " + fid); 36 | } 37 | try { 38 | return Long.parseLong(fid.substring(0, pos)); 39 | } catch (NumberFormatException nfe) { 40 | throw new IllegalArgumentException("Cannot parse fid: " + fid, nfe); 41 | } 42 | } 43 | 44 | @Override 45 | public String toString() { 46 | return "WeedFSFile [fid=" + fid + ", version=" + version + "]"; 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/test/java/com/scoopit/weedfs/benchmark/StatsCollector.java: -------------------------------------------------------------------------------- 1 | package com.scoopit.weedfs.benchmark; 2 | 3 | import akka.actor.Props; 4 | import akka.actor.UntypedActor; 5 | 6 | public class StatsCollector extends UntypedActor { 7 | 8 | public static enum Event { 9 | noLocation, wrongChecksum, writeFile, readFile, printStats, wrongUploadedSize 10 | } 11 | 12 | int noLocation, wrongChecksum, writeFile, readFile, wrongUploadedSize; 13 | 14 | public static Props mkProps() { 15 | return Props.create(StatsCollector.class); 16 | } 17 | 18 | @Override 19 | public void onReceive(Object arg0) throws Exception { 20 | Event event = (Event) arg0; 21 | switch(event){ 22 | case noLocation: 23 | noLocation++; 24 | break; 25 | case wrongChecksum: 26 | wrongChecksum++; 27 | break; 28 | case writeFile: 29 | writeFile++; 30 | break; 31 | case readFile: 32 | readFile++; 33 | break; 34 | case wrongUploadedSize: 35 | wrongUploadedSize++; 36 | break; 37 | case printStats: 38 | System.out.println("file read: " + readFile + ", file writes: " + writeFile + ", checksum errors: " + wrongChecksum 39 | + ", wrong uploaded size: " + wrongUploadedSize 40 | + ", no location error: " + noLocation); 41 | break; 42 | default: throw new IllegalArgumentException("Unknown event: "+event); 43 | } 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/test/java/com/scoopit/weedfs/benchmark/LoadTest.java: -------------------------------------------------------------------------------- 1 | package com.scoopit.weedfs.benchmark; 2 | 3 | import java.net.MalformedURLException; 4 | import java.net.URL; 5 | import java.util.Random; 6 | import java.util.concurrent.ConcurrentHashMap; 7 | 8 | import akka.actor.ActorRef; 9 | import akka.actor.ActorSystem; 10 | import akka.routing.RoundRobinRouter; 11 | 12 | /** 13 | * Have fun with Akka actors... 14 | */ 15 | public class LoadTest { 16 | 17 | /** Map */ 18 | static final ConcurrentHashMap fsDb = new ConcurrentHashMap<>(100000); 19 | 20 | static Random random = new Random(); 21 | 22 | static final URL MASTER_URL; 23 | static { 24 | try { 25 | MASTER_URL = new URL("http://homer:9333"); 26 | } catch (MalformedURLException u) { 27 | throw new RuntimeException(u); 28 | } 29 | } 30 | 31 | public static void main(String[] args) throws Exception { 32 | ActorSystem s = ActorSystem.create(); 33 | 34 | ActorRef stats = s.actorOf(StatsCollector.mkProps(), "stats"); 35 | ActorRef upload = s.actorOf(UploadRandomFile.mkProps(stats).withRouter(new RoundRobinRouter(100)), "upload"); 36 | ActorRef read = s.actorOf(ReadAndCheckFile.mkProps(stats).withRouter(new RoundRobinRouter(100)), "read"); 37 | ActorRef maestro = s.actorOf(Maestro.mkProps(upload, read), "maestro"); 38 | 39 | long t = System.currentTimeMillis(); 40 | while (true) { 41 | Thread.sleep(random.nextInt(10)); 42 | 43 | if (System.currentTimeMillis() - t > 1000) { 44 | stats.tell(StatsCollector.Event.printStats, ActorRef.noSender()); 45 | t = System.currentTimeMillis(); 46 | } 47 | 48 | maestro.tell(new Object(), ActorRef.noSender()); 49 | 50 | } 51 | 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/AssignParams.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2013 Scoop IT SAS (http://scoop.it/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Philippe GASSMANN 18 | * Jean-Baptiste BELLET 19 | */ 20 | package com.scoopit.weedfs.client; 21 | 22 | public class AssignParams { 23 | 24 | final ReplicationStrategy replicationStrategy; 25 | 26 | final int versionCount; 27 | 28 | final String collection; 29 | 30 | public static final AssignParams DEFAULT = new AssignParams(); 31 | 32 | public AssignParams() { 33 | this(null, 1, null); 34 | } 35 | 36 | public AssignParams(int versionCount) { 37 | this(null, versionCount, null); 38 | } 39 | 40 | public AssignParams(ReplicationStrategy replicationStrategy) { 41 | this(null, 1, replicationStrategy); 42 | } 43 | 44 | public AssignParams(String collection, ReplicationStrategy replicationStrategy) { 45 | this(collection, 1, replicationStrategy); 46 | } 47 | 48 | public AssignParams(String collection, int versionCount, ReplicationStrategy replicationStrategy) { 49 | this.collection = collection; 50 | this.versionCount = versionCount; 51 | this.replicationStrategy = replicationStrategy; 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/test/java/com/scoopit/weedfs/benchmark/ReadAndCheckFile.java: -------------------------------------------------------------------------------- 1 | package com.scoopit.weedfs.benchmark; 2 | 3 | import java.io.InputStream; 4 | import java.security.MessageDigest; 5 | import java.util.List; 6 | 7 | import akka.actor.ActorRef; 8 | import akka.actor.Props; 9 | import akka.actor.UntypedActor; 10 | 11 | import com.scoopit.weedfs.client.Location; 12 | import com.scoopit.weedfs.client.WeedFSClient; 13 | import com.scoopit.weedfs.client.WeedFSClientBuilder; 14 | import com.scoopit.weedfs.client.WeedFSFile; 15 | 16 | public class ReadAndCheckFile extends UntypedActor { 17 | 18 | final ActorRef statsActor; 19 | 20 | public static Props mkProps(ActorRef statsActor) { 21 | return Props.create(ReadAndCheckFile.class, statsActor); 22 | } 23 | 24 | public ReadAndCheckFile(ActorRef statsActor) { 25 | this.statsActor = statsActor; 26 | } 27 | 28 | @Override 29 | public void onReceive(Object arg0) throws Exception { 30 | statsActor.tell(StatsCollector.Event.readFile, getSelf()); 31 | String fid = (String) arg0; 32 | 33 | WeedFSFile file = new WeedFSFile(fid); 34 | 35 | WeedFSClient client = WeedFSClientBuilder.createBuilder().setMasterUrl(LoadTest.MASTER_URL).build(); 36 | 37 | List locations = client.lookup(file.getVolumeId()); 38 | 39 | if (locations.size() == 0) { 40 | statsActor.tell(StatsCollector.Event.noLocation, getSelf()); 41 | return; 42 | } 43 | 44 | InputStream is = client.read(file, locations.get(0)); 45 | 46 | MessageDigest md5 = MessageDigest.getInstance("MD5"); 47 | 48 | byte buf[] = new byte[1024]; 49 | int n; 50 | while ((n = is.read(buf)) > 0) { 51 | md5.update(buf, 0, n); 52 | } 53 | is.close(); 54 | 55 | byte[] digest = md5.digest(); 56 | byte[] dbDigest = LoadTest.fsDb.get(fid); 57 | for (int i = 0; i < digest.length; i++) { 58 | if (digest[i] != dbDigest[i]) { 59 | statsActor.tell(StatsCollector.Event.wrongChecksum, getSelf()); 60 | return; 61 | } 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/status/Topology.java: -------------------------------------------------------------------------------- 1 | package com.scoopit.weedfs.client.status; 2 | 3 | import java.util.Collections; 4 | import java.util.LinkedList; 5 | import java.util.List; 6 | 7 | public class Topology extends AbstractNode { 8 | public List DataCenters; 9 | public List layouts; 10 | 11 | private class Stats { 12 | int dcCount; 13 | int rackCount; 14 | int nodeCount; 15 | List nodeList = new LinkedList<>(); 16 | } 17 | 18 | private Stats stats; 19 | 20 | private void computeStats() { 21 | if (DataCenters == null) { 22 | return; 23 | } 24 | stats = new Stats(); 25 | for (DataCenter dc : DataCenters) { 26 | stats.dcCount++; 27 | if (dc.Racks == null) { 28 | continue; 29 | } 30 | for (Rack rack : dc.Racks) { 31 | stats.rackCount++; 32 | if (rack.DataNodes == null) { 33 | continue; 34 | } 35 | stats.nodeCount += rack.DataNodes.size(); 36 | stats.nodeList.addAll(rack.DataNodes); 37 | } 38 | } 39 | } 40 | 41 | public int getDataCenterCount() { 42 | if (stats == null) { 43 | computeStats(); 44 | } 45 | if (stats == null) { 46 | return 0; 47 | } 48 | return stats.dcCount; 49 | } 50 | 51 | public int getRackCount() { 52 | if (stats == null) { 53 | computeStats(); 54 | } 55 | if (stats == null) { 56 | return 0; 57 | } 58 | return stats.rackCount; 59 | } 60 | 61 | public int getDataNodeCount() { 62 | if (stats == null) { 63 | computeStats(); 64 | } 65 | if (stats == null) { 66 | return 0; 67 | } 68 | return stats.nodeCount; 69 | } 70 | 71 | public List getDataNodes() { 72 | if (stats == null) { 73 | computeStats(); 74 | } 75 | if (stats == null) { 76 | return Collections.emptyList(); 77 | } 78 | return stats.nodeList; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/test/java/com/scoopit/weedfs/benchmark/UploadRandomFile.java: -------------------------------------------------------------------------------- 1 | package com.scoopit.weedfs.benchmark; 2 | 3 | import java.io.File; 4 | import java.io.FileInputStream; 5 | import java.io.FileOutputStream; 6 | import java.security.MessageDigest; 7 | import java.util.Random; 8 | 9 | import akka.actor.ActorRef; 10 | import akka.actor.Props; 11 | import akka.actor.UntypedActor; 12 | 13 | import com.scoopit.weedfs.client.AssignParams; 14 | import com.scoopit.weedfs.client.Assignation; 15 | import com.scoopit.weedfs.client.ReplicationStrategy; 16 | import com.scoopit.weedfs.client.WeedFSClient; 17 | import com.scoopit.weedfs.client.WeedFSClientBuilder; 18 | 19 | public class UploadRandomFile extends UntypedActor { 20 | 21 | Random random = new Random(); 22 | 23 | final ActorRef statsActor; 24 | 25 | public static Props mkProps(ActorRef statsActor) { 26 | return Props.create(UploadRandomFile.class, statsActor); 27 | } 28 | 29 | public UploadRandomFile(ActorRef statsActor){ 30 | this.statsActor = statsActor; 31 | } 32 | 33 | @Override 34 | public void onReceive(Object arg0) throws Exception { 35 | File f = File.createTempFile("weedfs-load-test", "tmp"); 36 | f.deleteOnExit(); 37 | FileOutputStream fos = new FileOutputStream(f); 38 | byte[] buf = new byte[random.nextInt(100) + 50]; 39 | 40 | MessageDigest md5 = MessageDigest.getInstance("MD5"); 41 | 42 | int size = 0; 43 | for (int i = 0; i < random.nextInt(1000000) + 1; i++) { 44 | random.nextBytes(buf); 45 | fos.write(buf); 46 | md5.update(buf); 47 | size+=buf.length; 48 | } 49 | fos.close(); 50 | 51 | WeedFSClient client = WeedFSClientBuilder.createBuilder().setMasterUrl(LoadTest.MASTER_URL).build(); 52 | Assignation a = client.assign(new AssignParams("java-loadtest", ReplicationStrategy.None)); 53 | // System.out.println(a.weedFSFile.fid + " assigned"); 54 | int writtenSize = client.write(a.weedFSFile, a.location, new FileInputStream(f), "someName"); 55 | 56 | if (writtenSize != size) { 57 | statsActor.tell(StatsCollector.Event.wrongUploadedSize, getSelf()); 58 | return; 59 | } 60 | 61 | // System.out.println(a.weedFSFile.fid + " written, size=" + size); 62 | // update database 63 | LoadTest.fsDb.put(a.weedFSFile.fid, md5.digest()); 64 | 65 | statsActor.tell(StatsCollector.Event.writeFile, getSelf()); 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/ReplicationStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2013 Scoop IT SAS (http://scoop.it/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Philippe GASSMANN 18 | * Jean-Baptiste BELLET 19 | */ 20 | package com.scoopit.weedfs.client; 21 | 22 | public enum ReplicationStrategy { 23 | /** 24 | * no replication 25 | */ 26 | None("000"), 27 | /** 28 | * replicate once on the same rack 29 | */ 30 | OnceOnSameRack("001"), 31 | /** 32 | * replicate once on a different rack, but same data center 33 | */ 34 | OnceOnDifferentRack("010"), 35 | /** 36 | * replicate once on a different data center 37 | */ 38 | OnceOnDifferentDC("100"), 39 | /** 40 | * replicate twice on two different data center 41 | */ 42 | TwiceOnDifferentDC("200"), 43 | /** 44 | * replicate once on a different rack, and once on a different data center 45 | */ 46 | OnceOnDifferentRackAndOnceOnDifferentDC("110"); 47 | 48 | private ReplicationStrategy(String parameterValue) { 49 | this.parameterValue = parameterValue; 50 | } 51 | 52 | final String parameterValue; 53 | 54 | public static ReplicationStrategy fromParameterValue(String parameterValue) { 55 | // yes this is ugly 56 | switch (parameterValue) { 57 | case "000": 58 | return None; 59 | case "001": 60 | return OnceOnSameRack; 61 | case "010": 62 | return OnceOnDifferentRack; 63 | case "100": 64 | return OnceOnDifferentDC; 65 | case "200": 66 | return TwiceOnDifferentDC; 67 | case "110": 68 | return OnceOnDifferentRackAndOnceOnDifferentDC; 69 | default: 70 | throw new IllegalArgumentException("Unknown Replication Strategy: " + parameterValue); 71 | } 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/WeedFSClientBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2013 Scoop IT SAS (http://scoop.it/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Philippe GASSMANN 18 | * Jean-Baptiste BELLET 19 | */ 20 | package com.scoopit.weedfs.client; 21 | 22 | import java.net.MalformedURLException; 23 | import java.net.URL; 24 | 25 | import org.apache.http.client.HttpClient; 26 | import org.apache.http.impl.client.HttpClientBuilder; 27 | 28 | import com.scoopit.weedfs.client.caching.LookupCache; 29 | 30 | public class WeedFSClientBuilder { 31 | 32 | HttpClient httpClient; 33 | 34 | URL masterUrl; 35 | 36 | LookupCache lookupCache; 37 | 38 | public WeedFSClientBuilder() { 39 | 40 | } 41 | 42 | public static WeedFSClientBuilder createBuilder() { 43 | return new WeedFSClientBuilder(); 44 | } 45 | 46 | public WeedFSClientBuilder setHttpClient(HttpClient httpClient) { 47 | this.httpClient = httpClient; 48 | return this; 49 | } 50 | 51 | public WeedFSClientBuilder setMasterUrl(URL masterUrl) { 52 | this.masterUrl = masterUrl; 53 | return this; 54 | } 55 | 56 | public WeedFSClientBuilder setLookupCache(LookupCache lookupCache) { 57 | this.lookupCache = lookupCache; 58 | return this; 59 | } 60 | 61 | public WeedFSClient build() { 62 | if (masterUrl == null) { 63 | try { 64 | // default url for testing purpose 65 | masterUrl = new URL("http://localhost:9333"); 66 | } catch (MalformedURLException e) { 67 | // This cannot happen by construction 68 | throw new Error(e); 69 | } 70 | } 71 | 72 | if (httpClient == null) { 73 | // minimal http client 74 | httpClient = HttpClientBuilder.create().build(); 75 | } 76 | 77 | return new WeedFSClientImpl(masterUrl, httpClient, lookupCache); 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/WeedFSClientMock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2013 Scoop IT SAS (http://scoop.it/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Philippe GASSMANN 18 | * Jean-Baptiste BELLET 19 | */ 20 | package com.scoopit.weedfs.client; 21 | 22 | import java.io.File; 23 | import java.io.IOException; 24 | import java.io.InputStream; 25 | import java.util.List; 26 | 27 | import com.scoopit.weedfs.client.status.MasterStatus; 28 | import com.scoopit.weedfs.client.status.VolumeStatus; 29 | 30 | public class WeedFSClientMock implements WeedFSClient { 31 | 32 | @Override 33 | public Assignation assign(AssignParams params) throws IOException, WeedFSException { 34 | return null; 35 | } 36 | 37 | @Override 38 | public int write(WeedFSFile weedFSFile, Location location, File file) throws IOException, WeedFSException { 39 | return 0; 40 | } 41 | 42 | @Override 43 | public int write(WeedFSFile file, Location location, byte[] dataToUpload, String fileName) throws IOException, 44 | WeedFSException { 45 | return 0; 46 | } 47 | 48 | @Override 49 | public int write(WeedFSFile file, Location location, InputStream inputToUpload, String fileName) 50 | throws IOException, WeedFSException { 51 | return 0; 52 | } 53 | 54 | @Override 55 | public void delete(WeedFSFile file, Location location) throws IOException, WeedFSException { 56 | } 57 | 58 | @Override 59 | public List lookup(long volumeId) throws IOException, WeedFSException { 60 | return null; 61 | } 62 | 63 | @Override 64 | public InputStream read(WeedFSFile file, Location location) throws IOException, WeedFSException { 65 | return null; 66 | } 67 | 68 | @Override 69 | public MasterStatus getMasterStatus() { 70 | return null; 71 | } 72 | 73 | @Override 74 | public VolumeStatus getVolumeStatus(Location location) { 75 | return null; 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | com.scoopit 6 | weed-fs-client 7 | 1.0.3 8 | jar 9 | 10 | weed-fs-client 11 | http://maven.apache.org 12 | 13 | 14 | UTF-8 15 | 16 | 17 | 18 | 19 | junit 20 | junit 21 | 3.8.1 22 | test 23 | 24 | 25 | junit 26 | junit 27 | 3.8.1 28 | test 29 | 30 | 31 | com.typesafe.akka 32 | akka-actor_2.10 33 | 2.2.0 34 | test 35 | 36 | 37 | com.fasterxml.jackson.core 38 | jackson-core 39 | 2.2.3 40 | 41 | 42 | 43 | org.apache.httpcomponents 44 | httpclient 45 | 4.3.1 46 | 47 | 48 | org.apache.httpcomponents 49 | httpmime 50 | 4.3.1 51 | 52 | 53 | com.fasterxml.jackson.core 54 | jackson-databind 55 | 2.2.2 56 | 57 | 58 | commons-lang 59 | commons-lang 60 | 2.3 61 | 62 | 63 | 64 | 65 | 66 | org.apache.maven.plugins 67 | maven-compiler-plugin 68 | 3.0 69 | 70 | 1.7 71 | 1.7 72 | 73 | 74 | 75 | org.apache.maven.plugins 76 | maven-source-plugin 77 | 2.2.1 78 | 79 | 80 | attach-sources 81 | 82 | jar 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/WeedFSClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2013 Scoop IT SAS (http://scoop.it/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Philippe GASSMANN 18 | * Jean-Baptiste BELLET 19 | */ 20 | package com.scoopit.weedfs.client; 21 | 22 | import java.io.File; 23 | import java.io.IOException; 24 | import java.io.InputStream; 25 | import java.util.List; 26 | 27 | import com.scoopit.weedfs.client.status.MasterStatus; 28 | import com.scoopit.weedfs.client.status.VolumeStatus; 29 | 30 | /** 31 | * 
32 |  *                      ,
33 |  *                     dM
34 |  *                     MMr
35 |  *                    4MMML                  .
36 |  *                    MMMMM.                xf
37 |  *    .              "M6MMM               .MM-
38 |  *     Mh..          +MM5MMM            .MMMM
39 |  *     .MMM.         .MMMMML.          MMMMMh
40 |  *      )MMMh.        MM5MMM         MMMMMMM
41 |  *       3MMMMx.     'MMM3MMf      xnMMMMMM"
42 |  *       '*MMMMM      MMMMMM.     nMMMMMMP"
43 |  *         *MMMMMx    "MMM5M\    .MMMMMMM=
44 |  *          *MMMMMh   "MMMMM"   JMMMMMMP
45 |  *            MMMMMM   GMMMM.  dMMMMMM            .
46 |  *             MMMMMM  "MMMM  .MMMMM(        .nnMP"
47 |  *  ..          *MMMMx  MMM"  dMMMM"    .nnMMMMM*
48 |  *   "MMn...     'MMMMr 'MM   MMM"   .nMMMMMMM*"
49 |  *    "4MMMMnn..   *MMM  MM  MMP"  .dMMMMMMM""
50 |  *      ^MMMMMMMMx.  *ML "M .M*  .MMMMMM**"
51 |  *         *PMMMMMMhn. *x > M  .MMMM**""
52 |  *            ""**MMMMhx/.h/ .=*"
53 |  *                     .3P"%....
54 |  *                   nP"     "*MMnx
55 |  * 
56 |  *
57 | * 58 | * Note: fileName that exceeds 256 characters will be truncated. 59 | */ 60 | public interface WeedFSClient { 61 | Assignation assign(AssignParams params) throws IOException, WeedFSException; 62 | 63 | int write(WeedFSFile weedFSFile, Location location, File file) throws IOException, WeedFSException; 64 | 65 | int write(WeedFSFile file, Location location, byte[] dataToUpload, String fileName) throws IOException, WeedFSException; 66 | 67 | int write(WeedFSFile file, Location location, InputStream inputToUpload, String fileName) throws IOException, WeedFSException; 68 | 69 | void delete(WeedFSFile file, Location location) throws IOException, WeedFSException; 70 | 71 | List lookup(long volumeId) throws IOException, WeedFSException; 72 | 73 | InputStream read(WeedFSFile file, Location location) throws IOException, WeedFSException; 74 | 75 | MasterStatus getMasterStatus() throws IOException; 76 | 77 | VolumeStatus getVolumeStatus(Location location) throws IOException; 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/com/scoopit/weedfs/client/WeedFSClientImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) Copyright 2013 Scoop IT SAS (http://scoop.it/) and others. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | * 16 | * Contributors: 17 | * Philippe GASSMANN 18 | * Jean-Baptiste BELLET 19 | */ 20 | package com.scoopit.weedfs.client; 21 | 22 | import java.io.File; 23 | import java.io.IOException; 24 | import java.io.InputStream; 25 | import java.net.URL; 26 | import java.util.List; 27 | 28 | import org.apache.commons.lang.StringUtils; 29 | import org.apache.http.HttpResponse; 30 | import org.apache.http.StatusLine; 31 | import org.apache.http.client.HttpClient; 32 | import org.apache.http.client.methods.HttpDelete; 33 | import org.apache.http.client.methods.HttpGet; 34 | import org.apache.http.client.methods.HttpPost; 35 | import org.apache.http.entity.ContentType; 36 | import org.apache.http.entity.mime.MultipartEntityBuilder; 37 | 38 | import com.fasterxml.jackson.core.JsonParseException; 39 | import com.fasterxml.jackson.databind.JsonMappingException; 40 | import com.fasterxml.jackson.databind.ObjectMapper; 41 | import com.scoopit.weedfs.client.caching.LookupCache; 42 | import com.scoopit.weedfs.client.net.AssignResult; 43 | import com.scoopit.weedfs.client.net.LookupResult; 44 | import com.scoopit.weedfs.client.net.WriteResult; 45 | import com.scoopit.weedfs.client.status.MasterStatus; 46 | import com.scoopit.weedfs.client.status.VolumeStatus; 47 | 48 | class WeedFSClientImpl implements WeedFSClient { 49 | 50 | final URL masterURL; 51 | final HttpClient httpClient; 52 | final LookupCache lookupCache; 53 | 54 | WeedFSClientImpl(URL masterURL, HttpClient httpClient, LookupCache lookupCache) { 55 | this.masterURL = masterURL; 56 | this.httpClient = httpClient; 57 | this.lookupCache = lookupCache; 58 | } 59 | 60 | @Override 61 | public Assignation assign(AssignParams params) throws IOException, WeedFSException { 62 | StringBuilder url = new StringBuilder(new URL(masterURL, "/dir/assign").toExternalForm()); 63 | url.append("?count="); 64 | url.append(params.versionCount); 65 | if (params.replicationStrategy != null) { 66 | url.append("&replication="); 67 | url.append(params.replicationStrategy.parameterValue); 68 | } 69 | 70 | if (params.collection != null) { 71 | url.append("&collection="); 72 | url.append(params.collection); 73 | } 74 | 75 | HttpGet get = new HttpGet(url.toString()); 76 | try { 77 | HttpResponse response = httpClient.execute(get); 78 | 79 | ObjectMapper mapper = new ObjectMapper(); 80 | try { 81 | AssignResult result = mapper.readValue(response.getEntity().getContent(), AssignResult.class); 82 | 83 | if (result.error != null) { 84 | throw new WeedFSException(result.error); 85 | } 86 | 87 | return new Assignation(result); 88 | } catch (JsonMappingException | JsonParseException e) { 89 | throw new WeedFSException("Unable to parse JSON from weed-fs", e); 90 | } 91 | } finally { 92 | get.abort(); 93 | } 94 | } 95 | 96 | @Override 97 | public void delete(WeedFSFile file, Location location) throws IOException, WeedFSException { 98 | StringBuilder url = new StringBuilder(); 99 | if (!location.publicUrl.contains("http")) { 100 | url.append("http://"); 101 | } 102 | url.append(location.publicUrl); 103 | url.append("/"); 104 | url.append(file.fid); 105 | 106 | HttpDelete delete = new HttpDelete(url.toString()); 107 | try { 108 | HttpResponse response = httpClient.execute(delete); 109 | 110 | StatusLine line = response.getStatusLine(); 111 | if (line.getStatusCode() < 200 || line.getStatusCode() > 299) { 112 | throw new WeedFSException("Error deleting file " + file.fid + " on " + location.publicUrl + ": " + line.getStatusCode() + " " 113 | + line.getReasonPhrase()); 114 | } 115 | } finally { 116 | delete.abort(); 117 | } 118 | } 119 | 120 | @Override 121 | public List lookup(long volumeId) throws IOException, WeedFSException { 122 | if (lookupCache != null) { 123 | List ret = lookupCache.lookup(volumeId); 124 | if (ret != null) { 125 | return ret; 126 | } 127 | } 128 | 129 | StringBuilder url = new StringBuilder(new URL(masterURL, "/dir/lookup").toExternalForm()); 130 | url.append("?volumeId="); 131 | url.append(volumeId); 132 | 133 | HttpGet get = new HttpGet(url.toString()); 134 | try { 135 | HttpResponse response = httpClient.execute(get); 136 | 137 | ObjectMapper mapper = new ObjectMapper(); 138 | try { 139 | LookupResult result = mapper.readValue(response.getEntity().getContent(), LookupResult.class); 140 | 141 | if (result.error != null) { 142 | throw new WeedFSException(result.error); 143 | } 144 | 145 | if(lookupCache != null && result.locations != null && result.locations.size() > 0) { 146 | lookupCache.setLocation(volumeId, result.locations); 147 | } 148 | 149 | return result.locations; 150 | } catch (JsonMappingException | JsonParseException e) { 151 | throw new WeedFSException("Unable to parse JSON from weed-fs", e); 152 | } 153 | } finally { 154 | get.abort(); 155 | } 156 | 157 | } 158 | 159 | @Override 160 | public int write(WeedFSFile file, Location location, File fileToUpload) throws IOException, WeedFSException { 161 | if (fileToUpload.length() == 0) { 162 | throw new WeedFSException("Cannot write a 0-length file"); 163 | } 164 | return write(file, location, fileToUpload, null, null, null); 165 | } 166 | 167 | @Override 168 | public int write(WeedFSFile file, Location location, byte[] dataToUpload, String fileName) throws IOException, WeedFSException { 169 | if (dataToUpload.length == 0) { 170 | throw new WeedFSException("Cannot write a 0-length data"); 171 | } 172 | return write(file, location, null, dataToUpload, null, fileName); 173 | } 174 | 175 | @Override 176 | public int write(WeedFSFile file, Location location, InputStream inputToUpload, String fileName) throws IOException, WeedFSException { 177 | return write(file, location, null, null, inputToUpload, fileName); 178 | } 179 | 180 | private String sanitizeFileName(String fileName) { 181 | if (StringUtils.isBlank(fileName)) { 182 | return "file"; 183 | } else if (fileName.length() > 256) { 184 | return fileName.substring(0, 255); 185 | } 186 | return fileName; 187 | 188 | } 189 | 190 | private int write(WeedFSFile file, Location location, File fileToUpload, byte[] dataToUpload, InputStream inputToUpload, String fileName) 191 | throws IOException, WeedFSException { 192 | StringBuilder url = new StringBuilder(); 193 | if (!location.publicUrl.contains("http")) { 194 | url.append("http://"); 195 | } 196 | url.append(location.publicUrl); 197 | url.append('/'); 198 | url.append(file.fid); 199 | 200 | if (file.version > 0) { 201 | url.append('_'); 202 | url.append(file.version); 203 | } 204 | 205 | HttpPost post = new HttpPost(url.toString()); 206 | 207 | MultipartEntityBuilder multipartEntityBuilder = MultipartEntityBuilder.create(); 208 | if (fileToUpload != null) { 209 | if (fileName == null) { 210 | fileName = fileToUpload.getName(); 211 | } 212 | multipartEntityBuilder.addBinaryBody("file", fileToUpload, ContentType.APPLICATION_OCTET_STREAM, sanitizeFileName(fileName)); 213 | } else if (dataToUpload != null) { 214 | multipartEntityBuilder.addBinaryBody("file", dataToUpload, ContentType.APPLICATION_OCTET_STREAM, sanitizeFileName(fileName)); 215 | } else { 216 | multipartEntityBuilder.addBinaryBody("file", inputToUpload, ContentType.APPLICATION_OCTET_STREAM, sanitizeFileName(fileName)); 217 | } 218 | post.setEntity(multipartEntityBuilder.build()); 219 | 220 | try { 221 | HttpResponse response = httpClient.execute(post); 222 | ObjectMapper mapper = new ObjectMapper(); 223 | try { 224 | WriteResult result = mapper.readValue(response.getEntity().getContent(), WriteResult.class); 225 | 226 | if (result.error != null) { 227 | throw new WeedFSException(result.error); 228 | } 229 | 230 | return result.size; 231 | } catch (JsonMappingException | JsonParseException e) { 232 | throw new WeedFSException("Unable to parse JSON from weed-fs", e); 233 | } 234 | } finally { 235 | post.abort(); 236 | } 237 | } 238 | 239 | @Override 240 | public InputStream read(WeedFSFile file, Location location) throws IOException, WeedFSException, WeedFSFileNotFoundException { 241 | StringBuilder url = new StringBuilder(); 242 | if (!location.publicUrl.contains("http")) { 243 | url.append("http://"); 244 | } 245 | url.append(location.publicUrl); 246 | url.append('/'); 247 | url.append(file.fid); 248 | 249 | if (file.version > 0) { 250 | url.append('_'); 251 | url.append(file.version); 252 | } 253 | HttpGet get = new HttpGet(url.toString()); 254 | HttpResponse response = httpClient.execute(get); 255 | StatusLine line = response.getStatusLine(); 256 | if (line.getStatusCode() == 404) { 257 | get.abort(); 258 | throw new WeedFSFileNotFoundException(file, location); 259 | } 260 | if (line.getStatusCode() != 200) { 261 | get.abort(); 262 | throw new WeedFSException("Error reading file " + file.fid + " on " + location.publicUrl + ": " + line.getStatusCode() + " " 263 | + line.getReasonPhrase()); 264 | } 265 | return response.getEntity().getContent(); 266 | } 267 | 268 | @Override 269 | public MasterStatus getMasterStatus() throws IOException { 270 | URL url = new URL(masterURL, "/dir/status"); 271 | 272 | HttpGet get = new HttpGet(url.toString()); 273 | 274 | try { 275 | HttpResponse response = httpClient.execute(get); 276 | StatusLine line = response.getStatusLine(); 277 | 278 | if (line.getStatusCode() != 200) { 279 | throw new IOException("Not 200 status recieved for master status url: " + url.toExternalForm()); 280 | } 281 | 282 | ObjectMapper mapper = new ObjectMapper(); 283 | try { 284 | return mapper.readValue(response.getEntity().getContent(), MasterStatus.class); 285 | 286 | } catch (JsonMappingException | JsonParseException e) { 287 | throw new WeedFSException("Unable to parse JSON from weed-fs", e); 288 | } 289 | } finally { 290 | get.abort(); 291 | } 292 | } 293 | 294 | @Override 295 | public VolumeStatus getVolumeStatus(Location location) throws IOException { 296 | StringBuilder url = new StringBuilder(); 297 | if (!location.publicUrl.contains("http")) { 298 | url.append("http://"); 299 | } 300 | url.append(location.publicUrl); 301 | url.append("/status"); 302 | 303 | HttpGet get = new HttpGet(url.toString()); 304 | 305 | try { 306 | HttpResponse response = httpClient.execute(get); 307 | StatusLine line = response.getStatusLine(); 308 | 309 | if (line.getStatusCode() != 200) { 310 | throw new IOException("Not 200 status recieved for master status url: " + url.toString()); 311 | } 312 | 313 | ObjectMapper mapper = new ObjectMapper(); 314 | try { 315 | return mapper.readValue(response.getEntity().getContent(), VolumeStatus.class); 316 | 317 | } catch (JsonMappingException | JsonParseException e) { 318 | throw new WeedFSException("Unable to parse JSON from weed-fs", e); 319 | } 320 | } finally { 321 | get.abort(); 322 | } 323 | } 324 | } 325 | --------------------------------------------------------------------------------