├── .gitignore ├── README.md ├── go_hadoop_rpc ├── client.dat ├── hadoop_common │ ├── HAServiceProtocol.pb.go │ ├── IpcConnectionContext.pb.go │ ├── ProtobufRpcEngine.pb.go │ ├── ProtocolInfo.pb.go │ ├── RpcHeader.pb.go │ ├── Security.pb.go │ └── ZKFCProtocol.pb.go ├── main.go ├── main_test.go ├── proto_helper.go ├── server.dat └── writables.go ├── lib └── org │ └── apache │ └── hadoop │ └── hadoop-minikdc │ └── 3.0.0-SNAPSHOT │ ├── hadoop-minikdc-3.0.0-SNAPSHOT-sources.jar │ ├── hadoop-minikdc-3.0.0-SNAPSHOT.jar │ └── hadoop-minikdc-3.0.0-SNAPSHOT.pom ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── github │ │ └── elazar │ │ └── hadoop │ │ └── examples │ │ ├── HadoopBasicUsage.java │ │ ├── HadoopRPC.java │ │ └── TheProtocolProxy.java └── resources │ └── log4j.properties └── test └── java └── com └── github └── elazar └── hadoop └── examples ├── SaslGssapiExampleTest.java └── SaslMd5DigestExampleTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | go_hadoop_rpc/go_hadoop_rpc 3 | *.iml 4 | .*.swp 5 | .idea 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | = Hadoop 2 RPC mechanism Walktrhough 2 | 3 | This repository contains simple hadoop RPC interface, and a Go 4 | implementation that reads an RPC request. 5 | 6 | To run the Go server do 7 | 8 | $ cd go_hadoop_rpc 9 | $ go get code.google.com/p/goprotobuf/proto 10 | $ go build 11 | $ ./go_hadoop_rpc -server localhost:5121 12 | 13 | To run the Java RPC client 14 | 15 | $ mvn exec:java -Dexec.mainClass=com.github.elazar.hadoop.examples.HadoopRPC -Dexec.args="client" 16 | 17 | TO run the Java RPC server 18 | 19 | $ mvn exec:java -Dexec.mainClass=com.github.elazar.hadoop.examples.HadoopRPC -Dexec.args="server" 20 | 21 | and to run both client and server on the same process 22 | 23 | $ mvn exec:java -Dexec.mainClass=com.github.elazar.hadoop.examples.HadoopRPC 24 | -------------------------------------------------------------------------------- /go_hadoop_rpc/client.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elazarl/hadoop_rpc_walktrhough/1d9aabef04499aa7226d730d9411d8d898567c64/go_hadoop_rpc/client.dat -------------------------------------------------------------------------------- /go_hadoop_rpc/hadoop_common/HAServiceProtocol.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. 2 | // source: HAServiceProtocol.proto 3 | // DO NOT EDIT! 4 | 5 | package hadoop_common 6 | 7 | import proto "code.google.com/p/goprotobuf/proto" 8 | import json "encoding/json" 9 | import math "math" 10 | 11 | // Reference proto, json, and math imports to suppress error if they are not otherwise used. 12 | var _ = proto.Marshal 13 | var _ = &json.SyntaxError{} 14 | var _ = math.Inf 15 | 16 | type HAServiceStateProto int32 17 | 18 | const ( 19 | HAServiceStateProto_INITIALIZING HAServiceStateProto = 0 20 | HAServiceStateProto_ACTIVE HAServiceStateProto = 1 21 | HAServiceStateProto_STANDBY HAServiceStateProto = 2 22 | ) 23 | 24 | var HAServiceStateProto_name = map[int32]string{ 25 | 0: "INITIALIZING", 26 | 1: "ACTIVE", 27 | 2: "STANDBY", 28 | } 29 | var HAServiceStateProto_value = map[string]int32{ 30 | "INITIALIZING": 0, 31 | "ACTIVE": 1, 32 | "STANDBY": 2, 33 | } 34 | 35 | func (x HAServiceStateProto) Enum() *HAServiceStateProto { 36 | p := new(HAServiceStateProto) 37 | *p = x 38 | return p 39 | } 40 | func (x HAServiceStateProto) String() string { 41 | return proto.EnumName(HAServiceStateProto_name, int32(x)) 42 | } 43 | func (x HAServiceStateProto) MarshalJSON() ([]byte, error) { 44 | return json.Marshal(x.String()) 45 | } 46 | func (x *HAServiceStateProto) UnmarshalJSON(data []byte) error { 47 | value, err := proto.UnmarshalJSONEnum(HAServiceStateProto_value, data, "HAServiceStateProto") 48 | if err != nil { 49 | return err 50 | } 51 | *x = HAServiceStateProto(value) 52 | return nil 53 | } 54 | 55 | type HARequestSource int32 56 | 57 | const ( 58 | HARequestSource_REQUEST_BY_USER HARequestSource = 0 59 | HARequestSource_REQUEST_BY_USER_FORCED HARequestSource = 1 60 | HARequestSource_REQUEST_BY_ZKFC HARequestSource = 2 61 | ) 62 | 63 | var HARequestSource_name = map[int32]string{ 64 | 0: "REQUEST_BY_USER", 65 | 1: "REQUEST_BY_USER_FORCED", 66 | 2: "REQUEST_BY_ZKFC", 67 | } 68 | var HARequestSource_value = map[string]int32{ 69 | "REQUEST_BY_USER": 0, 70 | "REQUEST_BY_USER_FORCED": 1, 71 | "REQUEST_BY_ZKFC": 2, 72 | } 73 | 74 | func (x HARequestSource) Enum() *HARequestSource { 75 | p := new(HARequestSource) 76 | *p = x 77 | return p 78 | } 79 | func (x HARequestSource) String() string { 80 | return proto.EnumName(HARequestSource_name, int32(x)) 81 | } 82 | func (x HARequestSource) MarshalJSON() ([]byte, error) { 83 | return json.Marshal(x.String()) 84 | } 85 | func (x *HARequestSource) UnmarshalJSON(data []byte) error { 86 | value, err := proto.UnmarshalJSONEnum(HARequestSource_value, data, "HARequestSource") 87 | if err != nil { 88 | return err 89 | } 90 | *x = HARequestSource(value) 91 | return nil 92 | } 93 | 94 | type HAStateChangeRequestInfoProto struct { 95 | ReqSource *HARequestSource `protobuf:"varint,1,req,name=reqSource,enum=hadoop.common.HARequestSource" json:"reqSource,omitempty"` 96 | XXX_unrecognized []byte `json:"-"` 97 | } 98 | 99 | func (m *HAStateChangeRequestInfoProto) Reset() { *m = HAStateChangeRequestInfoProto{} } 100 | func (m *HAStateChangeRequestInfoProto) String() string { return proto.CompactTextString(m) } 101 | func (*HAStateChangeRequestInfoProto) ProtoMessage() {} 102 | 103 | func (m *HAStateChangeRequestInfoProto) GetReqSource() HARequestSource { 104 | if m != nil && m.ReqSource != nil { 105 | return *m.ReqSource 106 | } 107 | return 0 108 | } 109 | 110 | // * 111 | // void request 112 | type MonitorHealthRequestProto struct { 113 | XXX_unrecognized []byte `json:"-"` 114 | } 115 | 116 | func (m *MonitorHealthRequestProto) Reset() { *m = MonitorHealthRequestProto{} } 117 | func (m *MonitorHealthRequestProto) String() string { return proto.CompactTextString(m) } 118 | func (*MonitorHealthRequestProto) ProtoMessage() {} 119 | 120 | // * 121 | // void response 122 | type MonitorHealthResponseProto struct { 123 | XXX_unrecognized []byte `json:"-"` 124 | } 125 | 126 | func (m *MonitorHealthResponseProto) Reset() { *m = MonitorHealthResponseProto{} } 127 | func (m *MonitorHealthResponseProto) String() string { return proto.CompactTextString(m) } 128 | func (*MonitorHealthResponseProto) ProtoMessage() {} 129 | 130 | // * 131 | // void request 132 | type TransitionToActiveRequestProto struct { 133 | ReqInfo *HAStateChangeRequestInfoProto `protobuf:"bytes,1,req,name=reqInfo" json:"reqInfo,omitempty"` 134 | XXX_unrecognized []byte `json:"-"` 135 | } 136 | 137 | func (m *TransitionToActiveRequestProto) Reset() { *m = TransitionToActiveRequestProto{} } 138 | func (m *TransitionToActiveRequestProto) String() string { return proto.CompactTextString(m) } 139 | func (*TransitionToActiveRequestProto) ProtoMessage() {} 140 | 141 | func (m *TransitionToActiveRequestProto) GetReqInfo() *HAStateChangeRequestInfoProto { 142 | if m != nil { 143 | return m.ReqInfo 144 | } 145 | return nil 146 | } 147 | 148 | // * 149 | // void response 150 | type TransitionToActiveResponseProto struct { 151 | XXX_unrecognized []byte `json:"-"` 152 | } 153 | 154 | func (m *TransitionToActiveResponseProto) Reset() { *m = TransitionToActiveResponseProto{} } 155 | func (m *TransitionToActiveResponseProto) String() string { return proto.CompactTextString(m) } 156 | func (*TransitionToActiveResponseProto) ProtoMessage() {} 157 | 158 | // * 159 | // void request 160 | type TransitionToStandbyRequestProto struct { 161 | ReqInfo *HAStateChangeRequestInfoProto `protobuf:"bytes,1,req,name=reqInfo" json:"reqInfo,omitempty"` 162 | XXX_unrecognized []byte `json:"-"` 163 | } 164 | 165 | func (m *TransitionToStandbyRequestProto) Reset() { *m = TransitionToStandbyRequestProto{} } 166 | func (m *TransitionToStandbyRequestProto) String() string { return proto.CompactTextString(m) } 167 | func (*TransitionToStandbyRequestProto) ProtoMessage() {} 168 | 169 | func (m *TransitionToStandbyRequestProto) GetReqInfo() *HAStateChangeRequestInfoProto { 170 | if m != nil { 171 | return m.ReqInfo 172 | } 173 | return nil 174 | } 175 | 176 | // * 177 | // void response 178 | type TransitionToStandbyResponseProto struct { 179 | XXX_unrecognized []byte `json:"-"` 180 | } 181 | 182 | func (m *TransitionToStandbyResponseProto) Reset() { *m = TransitionToStandbyResponseProto{} } 183 | func (m *TransitionToStandbyResponseProto) String() string { return proto.CompactTextString(m) } 184 | func (*TransitionToStandbyResponseProto) ProtoMessage() {} 185 | 186 | // * 187 | // void request 188 | type GetServiceStatusRequestProto struct { 189 | XXX_unrecognized []byte `json:"-"` 190 | } 191 | 192 | func (m *GetServiceStatusRequestProto) Reset() { *m = GetServiceStatusRequestProto{} } 193 | func (m *GetServiceStatusRequestProto) String() string { return proto.CompactTextString(m) } 194 | func (*GetServiceStatusRequestProto) ProtoMessage() {} 195 | 196 | // * 197 | // Returns the state of the service 198 | type GetServiceStatusResponseProto struct { 199 | State *HAServiceStateProto `protobuf:"varint,1,req,name=state,enum=hadoop.common.HAServiceStateProto" json:"state,omitempty"` 200 | // If state is STANDBY, indicate whether it is 201 | // ready to become active. 202 | ReadyToBecomeActive *bool `protobuf:"varint,2,opt,name=readyToBecomeActive" json:"readyToBecomeActive,omitempty"` 203 | // If not ready to become active, a textual explanation of why not 204 | NotReadyReason *string `protobuf:"bytes,3,opt,name=notReadyReason" json:"notReadyReason,omitempty"` 205 | XXX_unrecognized []byte `json:"-"` 206 | } 207 | 208 | func (m *GetServiceStatusResponseProto) Reset() { *m = GetServiceStatusResponseProto{} } 209 | func (m *GetServiceStatusResponseProto) String() string { return proto.CompactTextString(m) } 210 | func (*GetServiceStatusResponseProto) ProtoMessage() {} 211 | 212 | func (m *GetServiceStatusResponseProto) GetState() HAServiceStateProto { 213 | if m != nil && m.State != nil { 214 | return *m.State 215 | } 216 | return 0 217 | } 218 | 219 | func (m *GetServiceStatusResponseProto) GetReadyToBecomeActive() bool { 220 | if m != nil && m.ReadyToBecomeActive != nil { 221 | return *m.ReadyToBecomeActive 222 | } 223 | return false 224 | } 225 | 226 | func (m *GetServiceStatusResponseProto) GetNotReadyReason() string { 227 | if m != nil && m.NotReadyReason != nil { 228 | return *m.NotReadyReason 229 | } 230 | return "" 231 | } 232 | 233 | func init() { 234 | proto.RegisterEnum("hadoop.common.HAServiceStateProto", HAServiceStateProto_name, HAServiceStateProto_value) 235 | proto.RegisterEnum("hadoop.common.HARequestSource", HARequestSource_name, HARequestSource_value) 236 | } 237 | -------------------------------------------------------------------------------- /go_hadoop_rpc/hadoop_common/IpcConnectionContext.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. 2 | // source: IpcConnectionContext.proto 3 | // DO NOT EDIT! 4 | 5 | package hadoop_common 6 | 7 | import proto "code.google.com/p/goprotobuf/proto" 8 | import json "encoding/json" 9 | import math "math" 10 | 11 | // Reference proto, json, and math imports to suppress error if they are not otherwise used. 12 | var _ = proto.Marshal 13 | var _ = &json.SyntaxError{} 14 | var _ = math.Inf 15 | 16 | // * 17 | // Spec for UserInformationProto is specified in ProtoUtil#makeIpcConnectionContext 18 | type UserInformationProto struct { 19 | EffectiveUser *string `protobuf:"bytes,1,opt,name=effectiveUser" json:"effectiveUser,omitempty"` 20 | RealUser *string `protobuf:"bytes,2,opt,name=realUser" json:"realUser,omitempty"` 21 | XXX_unrecognized []byte `json:"-"` 22 | } 23 | 24 | func (m *UserInformationProto) Reset() { *m = UserInformationProto{} } 25 | func (m *UserInformationProto) String() string { return proto.CompactTextString(m) } 26 | func (*UserInformationProto) ProtoMessage() {} 27 | 28 | func (m *UserInformationProto) GetEffectiveUser() string { 29 | if m != nil && m.EffectiveUser != nil { 30 | return *m.EffectiveUser 31 | } 32 | return "" 33 | } 34 | 35 | func (m *UserInformationProto) GetRealUser() string { 36 | if m != nil && m.RealUser != nil { 37 | return *m.RealUser 38 | } 39 | return "" 40 | } 41 | 42 | // * 43 | // The connection context is sent as part of the connection establishment. 44 | // It establishes the context for ALL Rpc calls within the connection. 45 | type IpcConnectionContextProto struct { 46 | // UserInfo beyond what is determined as part of security handshake 47 | // at connection time (kerberos, tokens etc). 48 | UserInfo *UserInformationProto `protobuf:"bytes,2,opt,name=userInfo" json:"userInfo,omitempty"` 49 | // Protocol name for next rpc layer. 50 | // The client created a proxy with this protocol name 51 | Protocol *string `protobuf:"bytes,3,opt,name=protocol" json:"protocol,omitempty"` 52 | XXX_unrecognized []byte `json:"-"` 53 | } 54 | 55 | func (m *IpcConnectionContextProto) Reset() { *m = IpcConnectionContextProto{} } 56 | func (m *IpcConnectionContextProto) String() string { return proto.CompactTextString(m) } 57 | func (*IpcConnectionContextProto) ProtoMessage() {} 58 | 59 | func (m *IpcConnectionContextProto) GetUserInfo() *UserInformationProto { 60 | if m != nil { 61 | return m.UserInfo 62 | } 63 | return nil 64 | } 65 | 66 | func (m *IpcConnectionContextProto) GetProtocol() string { 67 | if m != nil && m.Protocol != nil { 68 | return *m.Protocol 69 | } 70 | return "" 71 | } 72 | 73 | func init() { 74 | } 75 | -------------------------------------------------------------------------------- /go_hadoop_rpc/hadoop_common/ProtobufRpcEngine.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. 2 | // source: ProtobufRpcEngine.proto 3 | // DO NOT EDIT! 4 | 5 | package hadoop_common 6 | 7 | import proto "code.google.com/p/goprotobuf/proto" 8 | import json "encoding/json" 9 | import math "math" 10 | 11 | // Reference proto, json, and math imports to suppress error if they are not otherwise used. 12 | var _ = proto.Marshal 13 | var _ = &json.SyntaxError{} 14 | var _ = math.Inf 15 | 16 | // * 17 | // This message is the header for the Protobuf Rpc Engine 18 | // when sending a RPC request from RPC client to the RPC server. 19 | // The actual request (serialized as protobuf) follows this request. 20 | // 21 | // No special header is needed for the Rpc Response for Protobuf Rpc Engine. 22 | // The normal RPC response header (see RpcHeader.proto) are sufficient. 23 | type RequestHeaderProto struct { 24 | // * Name of the RPC method 25 | MethodName *string `protobuf:"bytes,1,req,name=methodName" json:"methodName,omitempty"` 26 | // * 27 | // RPCs for a particular interface (ie protocol) are done using a 28 | // IPC connection that is setup using rpcProxy. 29 | // The rpcProxy's has a declared protocol name that is 30 | // sent form client to server at connection time. 31 | // 32 | // Each Rpc call also sends a protocol name 33 | // (called declaringClassprotocolName). This name is usually the same 34 | // as the connection protocol name except in some cases. 35 | // For example metaProtocols such ProtocolInfoProto which get metainfo 36 | // about the protocol reuse the connection but need to indicate that 37 | // the actual protocol is different (i.e. the protocol is 38 | // ProtocolInfoProto) since they reuse the connection; in this case 39 | // the declaringClassProtocolName field is set to the ProtocolInfoProto 40 | DeclaringClassProtocolName *string `protobuf:"bytes,3,req,name=declaringClassProtocolName" json:"declaringClassProtocolName,omitempty"` 41 | // * protocol version of class declaring the called method 42 | ClientProtocolVersion *uint64 `protobuf:"varint,4,req,name=clientProtocolVersion" json:"clientProtocolVersion,omitempty"` 43 | XXX_unrecognized []byte `json:"-"` 44 | } 45 | 46 | func (m *RequestHeaderProto) Reset() { *m = RequestHeaderProto{} } 47 | func (m *RequestHeaderProto) String() string { return proto.CompactTextString(m) } 48 | func (*RequestHeaderProto) ProtoMessage() {} 49 | 50 | func (m *RequestHeaderProto) GetMethodName() string { 51 | if m != nil && m.MethodName != nil { 52 | return *m.MethodName 53 | } 54 | return "" 55 | } 56 | 57 | func (m *RequestHeaderProto) GetDeclaringClassProtocolName() string { 58 | if m != nil && m.DeclaringClassProtocolName != nil { 59 | return *m.DeclaringClassProtocolName 60 | } 61 | return "" 62 | } 63 | 64 | func (m *RequestHeaderProto) GetClientProtocolVersion() uint64 { 65 | if m != nil && m.ClientProtocolVersion != nil { 66 | return *m.ClientProtocolVersion 67 | } 68 | return 0 69 | } 70 | 71 | func init() { 72 | } 73 | -------------------------------------------------------------------------------- /go_hadoop_rpc/hadoop_common/ProtocolInfo.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. 2 | // source: ProtocolInfo.proto 3 | // DO NOT EDIT! 4 | 5 | package hadoop_common 6 | 7 | import proto "code.google.com/p/goprotobuf/proto" 8 | import json "encoding/json" 9 | import math "math" 10 | 11 | // Reference proto, json, and math imports to suppress error if they are not otherwise used. 12 | var _ = proto.Marshal 13 | var _ = &json.SyntaxError{} 14 | var _ = math.Inf 15 | 16 | // * 17 | // Request to get protocol versions for all supported rpc kinds. 18 | type GetProtocolVersionsRequestProto struct { 19 | Protocol *string `protobuf:"bytes,1,req,name=protocol" json:"protocol,omitempty"` 20 | XXX_unrecognized []byte `json:"-"` 21 | } 22 | 23 | func (m *GetProtocolVersionsRequestProto) Reset() { *m = GetProtocolVersionsRequestProto{} } 24 | func (m *GetProtocolVersionsRequestProto) String() string { return proto.CompactTextString(m) } 25 | func (*GetProtocolVersionsRequestProto) ProtoMessage() {} 26 | 27 | func (m *GetProtocolVersionsRequestProto) GetProtocol() string { 28 | if m != nil && m.Protocol != nil { 29 | return *m.Protocol 30 | } 31 | return "" 32 | } 33 | 34 | // * 35 | // Protocol version with corresponding rpc kind. 36 | type ProtocolVersionProto struct { 37 | RpcKind *string `protobuf:"bytes,1,req,name=rpcKind" json:"rpcKind,omitempty"` 38 | Versions []uint64 `protobuf:"varint,2,rep,name=versions" json:"versions,omitempty"` 39 | XXX_unrecognized []byte `json:"-"` 40 | } 41 | 42 | func (m *ProtocolVersionProto) Reset() { *m = ProtocolVersionProto{} } 43 | func (m *ProtocolVersionProto) String() string { return proto.CompactTextString(m) } 44 | func (*ProtocolVersionProto) ProtoMessage() {} 45 | 46 | func (m *ProtocolVersionProto) GetRpcKind() string { 47 | if m != nil && m.RpcKind != nil { 48 | return *m.RpcKind 49 | } 50 | return "" 51 | } 52 | 53 | func (m *ProtocolVersionProto) GetVersions() []uint64 { 54 | if m != nil { 55 | return m.Versions 56 | } 57 | return nil 58 | } 59 | 60 | // * 61 | // Get protocol version response. 62 | type GetProtocolVersionsResponseProto struct { 63 | ProtocolVersions []*ProtocolVersionProto `protobuf:"bytes,1,rep,name=protocolVersions" json:"protocolVersions,omitempty"` 64 | XXX_unrecognized []byte `json:"-"` 65 | } 66 | 67 | func (m *GetProtocolVersionsResponseProto) Reset() { *m = GetProtocolVersionsResponseProto{} } 68 | func (m *GetProtocolVersionsResponseProto) String() string { return proto.CompactTextString(m) } 69 | func (*GetProtocolVersionsResponseProto) ProtoMessage() {} 70 | 71 | func (m *GetProtocolVersionsResponseProto) GetProtocolVersions() []*ProtocolVersionProto { 72 | if m != nil { 73 | return m.ProtocolVersions 74 | } 75 | return nil 76 | } 77 | 78 | // * 79 | // Get protocol signature request. 80 | type GetProtocolSignatureRequestProto struct { 81 | Protocol *string `protobuf:"bytes,1,req,name=protocol" json:"protocol,omitempty"` 82 | RpcKind *string `protobuf:"bytes,2,req,name=rpcKind" json:"rpcKind,omitempty"` 83 | XXX_unrecognized []byte `json:"-"` 84 | } 85 | 86 | func (m *GetProtocolSignatureRequestProto) Reset() { *m = GetProtocolSignatureRequestProto{} } 87 | func (m *GetProtocolSignatureRequestProto) String() string { return proto.CompactTextString(m) } 88 | func (*GetProtocolSignatureRequestProto) ProtoMessage() {} 89 | 90 | func (m *GetProtocolSignatureRequestProto) GetProtocol() string { 91 | if m != nil && m.Protocol != nil { 92 | return *m.Protocol 93 | } 94 | return "" 95 | } 96 | 97 | func (m *GetProtocolSignatureRequestProto) GetRpcKind() string { 98 | if m != nil && m.RpcKind != nil { 99 | return *m.RpcKind 100 | } 101 | return "" 102 | } 103 | 104 | // * 105 | // Get protocol signature response. 106 | type GetProtocolSignatureResponseProto struct { 107 | ProtocolSignature []*ProtocolSignatureProto `protobuf:"bytes,1,rep,name=protocolSignature" json:"protocolSignature,omitempty"` 108 | XXX_unrecognized []byte `json:"-"` 109 | } 110 | 111 | func (m *GetProtocolSignatureResponseProto) Reset() { *m = GetProtocolSignatureResponseProto{} } 112 | func (m *GetProtocolSignatureResponseProto) String() string { return proto.CompactTextString(m) } 113 | func (*GetProtocolSignatureResponseProto) ProtoMessage() {} 114 | 115 | func (m *GetProtocolSignatureResponseProto) GetProtocolSignature() []*ProtocolSignatureProto { 116 | if m != nil { 117 | return m.ProtocolSignature 118 | } 119 | return nil 120 | } 121 | 122 | type ProtocolSignatureProto struct { 123 | Version *uint64 `protobuf:"varint,1,req,name=version" json:"version,omitempty"` 124 | Methods []uint32 `protobuf:"varint,2,rep,name=methods" json:"methods,omitempty"` 125 | XXX_unrecognized []byte `json:"-"` 126 | } 127 | 128 | func (m *ProtocolSignatureProto) Reset() { *m = ProtocolSignatureProto{} } 129 | func (m *ProtocolSignatureProto) String() string { return proto.CompactTextString(m) } 130 | func (*ProtocolSignatureProto) ProtoMessage() {} 131 | 132 | func (m *ProtocolSignatureProto) GetVersion() uint64 { 133 | if m != nil && m.Version != nil { 134 | return *m.Version 135 | } 136 | return 0 137 | } 138 | 139 | func (m *ProtocolSignatureProto) GetMethods() []uint32 { 140 | if m != nil { 141 | return m.Methods 142 | } 143 | return nil 144 | } 145 | 146 | func init() { 147 | } 148 | -------------------------------------------------------------------------------- /go_hadoop_rpc/hadoop_common/RpcHeader.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. 2 | // source: RpcHeader.proto 3 | // DO NOT EDIT! 4 | 5 | package hadoop_common 6 | 7 | import proto "code.google.com/p/goprotobuf/proto" 8 | import json "encoding/json" 9 | import math "math" 10 | 11 | // Reference proto, json, and math imports to suppress error if they are not otherwise used. 12 | var _ = proto.Marshal 13 | var _ = &json.SyntaxError{} 14 | var _ = math.Inf 15 | 16 | // * 17 | // RpcKind determine the rpcEngine and the serialization of the rpc request 18 | type RpcKindProto int32 19 | 20 | const ( 21 | RpcKindProto_RPC_BUILTIN RpcKindProto = 0 22 | RpcKindProto_RPC_WRITABLE RpcKindProto = 1 23 | RpcKindProto_RPC_PROTOCOL_BUFFER RpcKindProto = 2 24 | ) 25 | 26 | var RpcKindProto_name = map[int32]string{ 27 | 0: "RPC_BUILTIN", 28 | 1: "RPC_WRITABLE", 29 | 2: "RPC_PROTOCOL_BUFFER", 30 | } 31 | var RpcKindProto_value = map[string]int32{ 32 | "RPC_BUILTIN": 0, 33 | "RPC_WRITABLE": 1, 34 | "RPC_PROTOCOL_BUFFER": 2, 35 | } 36 | 37 | func (x RpcKindProto) Enum() *RpcKindProto { 38 | p := new(RpcKindProto) 39 | *p = x 40 | return p 41 | } 42 | func (x RpcKindProto) String() string { 43 | return proto.EnumName(RpcKindProto_name, int32(x)) 44 | } 45 | func (x RpcKindProto) MarshalJSON() ([]byte, error) { 46 | return json.Marshal(x.String()) 47 | } 48 | func (x *RpcKindProto) UnmarshalJSON(data []byte) error { 49 | value, err := proto.UnmarshalJSONEnum(RpcKindProto_value, data, "RpcKindProto") 50 | if err != nil { 51 | return err 52 | } 53 | *x = RpcKindProto(value) 54 | return nil 55 | } 56 | 57 | type RpcRequestHeaderProto_OperationProto int32 58 | 59 | const ( 60 | RpcRequestHeaderProto_RPC_FINAL_PACKET RpcRequestHeaderProto_OperationProto = 0 61 | RpcRequestHeaderProto_RPC_CONTINUATION_PACKET RpcRequestHeaderProto_OperationProto = 1 62 | RpcRequestHeaderProto_RPC_CLOSE_CONNECTION RpcRequestHeaderProto_OperationProto = 2 63 | ) 64 | 65 | var RpcRequestHeaderProto_OperationProto_name = map[int32]string{ 66 | 0: "RPC_FINAL_PACKET", 67 | 1: "RPC_CONTINUATION_PACKET", 68 | 2: "RPC_CLOSE_CONNECTION", 69 | } 70 | var RpcRequestHeaderProto_OperationProto_value = map[string]int32{ 71 | "RPC_FINAL_PACKET": 0, 72 | "RPC_CONTINUATION_PACKET": 1, 73 | "RPC_CLOSE_CONNECTION": 2, 74 | } 75 | 76 | func (x RpcRequestHeaderProto_OperationProto) Enum() *RpcRequestHeaderProto_OperationProto { 77 | p := new(RpcRequestHeaderProto_OperationProto) 78 | *p = x 79 | return p 80 | } 81 | func (x RpcRequestHeaderProto_OperationProto) String() string { 82 | return proto.EnumName(RpcRequestHeaderProto_OperationProto_name, int32(x)) 83 | } 84 | func (x RpcRequestHeaderProto_OperationProto) MarshalJSON() ([]byte, error) { 85 | return json.Marshal(x.String()) 86 | } 87 | func (x *RpcRequestHeaderProto_OperationProto) UnmarshalJSON(data []byte) error { 88 | value, err := proto.UnmarshalJSONEnum(RpcRequestHeaderProto_OperationProto_value, data, "RpcRequestHeaderProto_OperationProto") 89 | if err != nil { 90 | return err 91 | } 92 | *x = RpcRequestHeaderProto_OperationProto(value) 93 | return nil 94 | } 95 | 96 | type RpcResponseHeaderProto_RpcStatusProto int32 97 | 98 | const ( 99 | RpcResponseHeaderProto_SUCCESS RpcResponseHeaderProto_RpcStatusProto = 0 100 | RpcResponseHeaderProto_ERROR RpcResponseHeaderProto_RpcStatusProto = 1 101 | RpcResponseHeaderProto_FATAL RpcResponseHeaderProto_RpcStatusProto = 2 102 | ) 103 | 104 | var RpcResponseHeaderProto_RpcStatusProto_name = map[int32]string{ 105 | 0: "SUCCESS", 106 | 1: "ERROR", 107 | 2: "FATAL", 108 | } 109 | var RpcResponseHeaderProto_RpcStatusProto_value = map[string]int32{ 110 | "SUCCESS": 0, 111 | "ERROR": 1, 112 | "FATAL": 2, 113 | } 114 | 115 | func (x RpcResponseHeaderProto_RpcStatusProto) Enum() *RpcResponseHeaderProto_RpcStatusProto { 116 | p := new(RpcResponseHeaderProto_RpcStatusProto) 117 | *p = x 118 | return p 119 | } 120 | func (x RpcResponseHeaderProto_RpcStatusProto) String() string { 121 | return proto.EnumName(RpcResponseHeaderProto_RpcStatusProto_name, int32(x)) 122 | } 123 | func (x RpcResponseHeaderProto_RpcStatusProto) MarshalJSON() ([]byte, error) { 124 | return json.Marshal(x.String()) 125 | } 126 | func (x *RpcResponseHeaderProto_RpcStatusProto) UnmarshalJSON(data []byte) error { 127 | value, err := proto.UnmarshalJSONEnum(RpcResponseHeaderProto_RpcStatusProto_value, data, "RpcResponseHeaderProto_RpcStatusProto") 128 | if err != nil { 129 | return err 130 | } 131 | *x = RpcResponseHeaderProto_RpcStatusProto(value) 132 | return nil 133 | } 134 | 135 | type RpcResponseHeaderProto_RpcErrorCodeProto int32 136 | 137 | const ( 138 | // Non-fatal Rpc error - connection left open for future rpc calls 139 | RpcResponseHeaderProto_ERROR_APPLICATION RpcResponseHeaderProto_RpcErrorCodeProto = 1 140 | RpcResponseHeaderProto_ERROR_NO_SUCH_METHOD RpcResponseHeaderProto_RpcErrorCodeProto = 2 141 | RpcResponseHeaderProto_ERROR_NO_SUCH_PROTOCOL RpcResponseHeaderProto_RpcErrorCodeProto = 3 142 | RpcResponseHeaderProto_ERROR_RPC_SERVER RpcResponseHeaderProto_RpcErrorCodeProto = 4 143 | RpcResponseHeaderProto_ERROR_SERIALIZING_RESPONSE RpcResponseHeaderProto_RpcErrorCodeProto = 5 144 | RpcResponseHeaderProto_ERROR_RPC_VERSION_MISMATCH RpcResponseHeaderProto_RpcErrorCodeProto = 6 145 | // Fatal Server side Rpc error - connection closed 146 | RpcResponseHeaderProto_FATAL_UNKNOWN RpcResponseHeaderProto_RpcErrorCodeProto = 10 147 | RpcResponseHeaderProto_FATAL_UNSUPPORTED_SERIALIZATION RpcResponseHeaderProto_RpcErrorCodeProto = 11 148 | RpcResponseHeaderProto_FATAL_INVALID_RPC_HEADER RpcResponseHeaderProto_RpcErrorCodeProto = 12 149 | RpcResponseHeaderProto_FATAL_DESERIALIZING_REQUEST RpcResponseHeaderProto_RpcErrorCodeProto = 13 150 | RpcResponseHeaderProto_FATAL_VERSION_MISMATCH RpcResponseHeaderProto_RpcErrorCodeProto = 14 151 | RpcResponseHeaderProto_FATAL_UNAUTHORIZED RpcResponseHeaderProto_RpcErrorCodeProto = 15 152 | ) 153 | 154 | var RpcResponseHeaderProto_RpcErrorCodeProto_name = map[int32]string{ 155 | 1: "ERROR_APPLICATION", 156 | 2: "ERROR_NO_SUCH_METHOD", 157 | 3: "ERROR_NO_SUCH_PROTOCOL", 158 | 4: "ERROR_RPC_SERVER", 159 | 5: "ERROR_SERIALIZING_RESPONSE", 160 | 6: "ERROR_RPC_VERSION_MISMATCH", 161 | 10: "FATAL_UNKNOWN", 162 | 11: "FATAL_UNSUPPORTED_SERIALIZATION", 163 | 12: "FATAL_INVALID_RPC_HEADER", 164 | 13: "FATAL_DESERIALIZING_REQUEST", 165 | 14: "FATAL_VERSION_MISMATCH", 166 | 15: "FATAL_UNAUTHORIZED", 167 | } 168 | var RpcResponseHeaderProto_RpcErrorCodeProto_value = map[string]int32{ 169 | "ERROR_APPLICATION": 1, 170 | "ERROR_NO_SUCH_METHOD": 2, 171 | "ERROR_NO_SUCH_PROTOCOL": 3, 172 | "ERROR_RPC_SERVER": 4, 173 | "ERROR_SERIALIZING_RESPONSE": 5, 174 | "ERROR_RPC_VERSION_MISMATCH": 6, 175 | "FATAL_UNKNOWN": 10, 176 | "FATAL_UNSUPPORTED_SERIALIZATION": 11, 177 | "FATAL_INVALID_RPC_HEADER": 12, 178 | "FATAL_DESERIALIZING_REQUEST": 13, 179 | "FATAL_VERSION_MISMATCH": 14, 180 | "FATAL_UNAUTHORIZED": 15, 181 | } 182 | 183 | func (x RpcResponseHeaderProto_RpcErrorCodeProto) Enum() *RpcResponseHeaderProto_RpcErrorCodeProto { 184 | p := new(RpcResponseHeaderProto_RpcErrorCodeProto) 185 | *p = x 186 | return p 187 | } 188 | func (x RpcResponseHeaderProto_RpcErrorCodeProto) String() string { 189 | return proto.EnumName(RpcResponseHeaderProto_RpcErrorCodeProto_name, int32(x)) 190 | } 191 | func (x RpcResponseHeaderProto_RpcErrorCodeProto) MarshalJSON() ([]byte, error) { 192 | return json.Marshal(x.String()) 193 | } 194 | func (x *RpcResponseHeaderProto_RpcErrorCodeProto) UnmarshalJSON(data []byte) error { 195 | value, err := proto.UnmarshalJSONEnum(RpcResponseHeaderProto_RpcErrorCodeProto_value, data, "RpcResponseHeaderProto_RpcErrorCodeProto") 196 | if err != nil { 197 | return err 198 | } 199 | *x = RpcResponseHeaderProto_RpcErrorCodeProto(value) 200 | return nil 201 | } 202 | 203 | type RpcSaslProto_SaslState int32 204 | 205 | const ( 206 | RpcSaslProto_SUCCESS RpcSaslProto_SaslState = 0 207 | RpcSaslProto_NEGOTIATE RpcSaslProto_SaslState = 1 208 | RpcSaslProto_INITIATE RpcSaslProto_SaslState = 2 209 | RpcSaslProto_CHALLENGE RpcSaslProto_SaslState = 3 210 | RpcSaslProto_RESPONSE RpcSaslProto_SaslState = 4 211 | ) 212 | 213 | var RpcSaslProto_SaslState_name = map[int32]string{ 214 | 0: "SUCCESS", 215 | 1: "NEGOTIATE", 216 | 2: "INITIATE", 217 | 3: "CHALLENGE", 218 | 4: "RESPONSE", 219 | } 220 | var RpcSaslProto_SaslState_value = map[string]int32{ 221 | "SUCCESS": 0, 222 | "NEGOTIATE": 1, 223 | "INITIATE": 2, 224 | "CHALLENGE": 3, 225 | "RESPONSE": 4, 226 | } 227 | 228 | func (x RpcSaslProto_SaslState) Enum() *RpcSaslProto_SaslState { 229 | p := new(RpcSaslProto_SaslState) 230 | *p = x 231 | return p 232 | } 233 | func (x RpcSaslProto_SaslState) String() string { 234 | return proto.EnumName(RpcSaslProto_SaslState_name, int32(x)) 235 | } 236 | func (x RpcSaslProto_SaslState) MarshalJSON() ([]byte, error) { 237 | return json.Marshal(x.String()) 238 | } 239 | func (x *RpcSaslProto_SaslState) UnmarshalJSON(data []byte) error { 240 | value, err := proto.UnmarshalJSONEnum(RpcSaslProto_SaslState_value, data, "RpcSaslProto_SaslState") 241 | if err != nil { 242 | return err 243 | } 244 | *x = RpcSaslProto_SaslState(value) 245 | return nil 246 | } 247 | 248 | type RpcRequestHeaderProto struct { 249 | RpcKind *RpcKindProto `protobuf:"varint,1,opt,name=rpcKind,enum=hadoop.common.RpcKindProto" json:"rpcKind,omitempty"` 250 | RpcOp *RpcRequestHeaderProto_OperationProto `protobuf:"varint,2,opt,name=rpcOp,enum=hadoop.common.RpcRequestHeaderProto_OperationProto" json:"rpcOp,omitempty"` 251 | CallId *uint32 `protobuf:"varint,3,req,name=callId" json:"callId,omitempty"` 252 | ClientId []byte `protobuf:"bytes,4,req,name=clientId" json:"clientId,omitempty"` 253 | XXX_unrecognized []byte `json:"-"` 254 | } 255 | 256 | func (m *RpcRequestHeaderProto) Reset() { *m = RpcRequestHeaderProto{} } 257 | func (m *RpcRequestHeaderProto) String() string { return proto.CompactTextString(m) } 258 | func (*RpcRequestHeaderProto) ProtoMessage() {} 259 | 260 | func (m *RpcRequestHeaderProto) GetRpcKind() RpcKindProto { 261 | if m != nil && m.RpcKind != nil { 262 | return *m.RpcKind 263 | } 264 | return 0 265 | } 266 | 267 | func (m *RpcRequestHeaderProto) GetRpcOp() RpcRequestHeaderProto_OperationProto { 268 | if m != nil && m.RpcOp != nil { 269 | return *m.RpcOp 270 | } 271 | return 0 272 | } 273 | 274 | func (m *RpcRequestHeaderProto) GetCallId() uint32 { 275 | if m != nil && m.CallId != nil { 276 | return *m.CallId 277 | } 278 | return 0 279 | } 280 | 281 | func (m *RpcRequestHeaderProto) GetClientId() []byte { 282 | if m != nil { 283 | return m.ClientId 284 | } 285 | return nil 286 | } 287 | 288 | // * 289 | // Rpc Response Header 290 | // +------------------------------------------------------------------+ 291 | // | Rpc total response length in bytes (4 bytes int) | 292 | // | (sum of next two parts) | 293 | // +------------------------------------------------------------------+ 294 | // | RpcResponseHeaderProto - serialized delimited ie has len | 295 | // +------------------------------------------------------------------+ 296 | // | if request is successful: | 297 | // | - RpcResponse - The actual rpc response bytes follow | 298 | // | the response header | 299 | // | This response is serialized based on RpcKindProto | 300 | // | if request fails : | 301 | // | The rpc response header contains the necessary info | 302 | // +------------------------------------------------------------------+ 303 | // 304 | // Note that rpc response header is also used when connection setup fails. 305 | // Ie the response looks like a rpc response with a fake callId. 306 | type RpcResponseHeaderProto struct { 307 | CallId *uint32 `protobuf:"varint,1,req,name=callId" json:"callId,omitempty"` 308 | Status *RpcResponseHeaderProto_RpcStatusProto `protobuf:"varint,2,req,name=status,enum=hadoop.common.RpcResponseHeaderProto_RpcStatusProto" json:"status,omitempty"` 309 | ServerIpcVersionNum *uint32 `protobuf:"varint,3,opt,name=serverIpcVersionNum" json:"serverIpcVersionNum,omitempty"` 310 | ExceptionClassName *string `protobuf:"bytes,4,opt,name=exceptionClassName" json:"exceptionClassName,omitempty"` 311 | ErrorMsg *string `protobuf:"bytes,5,opt,name=errorMsg" json:"errorMsg,omitempty"` 312 | ErrorDetail *RpcResponseHeaderProto_RpcErrorCodeProto `protobuf:"varint,6,opt,name=errorDetail,enum=hadoop.common.RpcResponseHeaderProto_RpcErrorCodeProto" json:"errorDetail,omitempty"` 313 | XXX_unrecognized []byte `json:"-"` 314 | } 315 | 316 | func (m *RpcResponseHeaderProto) Reset() { *m = RpcResponseHeaderProto{} } 317 | func (m *RpcResponseHeaderProto) String() string { return proto.CompactTextString(m) } 318 | func (*RpcResponseHeaderProto) ProtoMessage() {} 319 | 320 | func (m *RpcResponseHeaderProto) GetCallId() uint32 { 321 | if m != nil && m.CallId != nil { 322 | return *m.CallId 323 | } 324 | return 0 325 | } 326 | 327 | func (m *RpcResponseHeaderProto) GetStatus() RpcResponseHeaderProto_RpcStatusProto { 328 | if m != nil && m.Status != nil { 329 | return *m.Status 330 | } 331 | return 0 332 | } 333 | 334 | func (m *RpcResponseHeaderProto) GetServerIpcVersionNum() uint32 { 335 | if m != nil && m.ServerIpcVersionNum != nil { 336 | return *m.ServerIpcVersionNum 337 | } 338 | return 0 339 | } 340 | 341 | func (m *RpcResponseHeaderProto) GetExceptionClassName() string { 342 | if m != nil && m.ExceptionClassName != nil { 343 | return *m.ExceptionClassName 344 | } 345 | return "" 346 | } 347 | 348 | func (m *RpcResponseHeaderProto) GetErrorMsg() string { 349 | if m != nil && m.ErrorMsg != nil { 350 | return *m.ErrorMsg 351 | } 352 | return "" 353 | } 354 | 355 | func (m *RpcResponseHeaderProto) GetErrorDetail() RpcResponseHeaderProto_RpcErrorCodeProto { 356 | if m != nil && m.ErrorDetail != nil { 357 | return *m.ErrorDetail 358 | } 359 | return 0 360 | } 361 | 362 | type RpcSaslProto struct { 363 | Version *uint32 `protobuf:"varint,1,opt,name=version" json:"version,omitempty"` 364 | State *RpcSaslProto_SaslState `protobuf:"varint,2,req,name=state,enum=hadoop.common.RpcSaslProto_SaslState" json:"state,omitempty"` 365 | Token []byte `protobuf:"bytes,3,opt,name=token" json:"token,omitempty"` 366 | Auths []*RpcSaslProto_SaslAuth `protobuf:"bytes,4,rep,name=auths" json:"auths,omitempty"` 367 | XXX_unrecognized []byte `json:"-"` 368 | } 369 | 370 | func (m *RpcSaslProto) Reset() { *m = RpcSaslProto{} } 371 | func (m *RpcSaslProto) String() string { return proto.CompactTextString(m) } 372 | func (*RpcSaslProto) ProtoMessage() {} 373 | 374 | func (m *RpcSaslProto) GetVersion() uint32 { 375 | if m != nil && m.Version != nil { 376 | return *m.Version 377 | } 378 | return 0 379 | } 380 | 381 | func (m *RpcSaslProto) GetState() RpcSaslProto_SaslState { 382 | if m != nil && m.State != nil { 383 | return *m.State 384 | } 385 | return 0 386 | } 387 | 388 | func (m *RpcSaslProto) GetToken() []byte { 389 | if m != nil { 390 | return m.Token 391 | } 392 | return nil 393 | } 394 | 395 | func (m *RpcSaslProto) GetAuths() []*RpcSaslProto_SaslAuth { 396 | if m != nil { 397 | return m.Auths 398 | } 399 | return nil 400 | } 401 | 402 | type RpcSaslProto_SaslAuth struct { 403 | Method *string `protobuf:"bytes,1,req,name=method" json:"method,omitempty"` 404 | Mechanism *string `protobuf:"bytes,2,req,name=mechanism" json:"mechanism,omitempty"` 405 | Protocol *string `protobuf:"bytes,3,opt,name=protocol" json:"protocol,omitempty"` 406 | ServerId *string `protobuf:"bytes,4,opt,name=serverId" json:"serverId,omitempty"` 407 | Challenge []byte `protobuf:"bytes,5,opt,name=challenge" json:"challenge,omitempty"` 408 | XXX_unrecognized []byte `json:"-"` 409 | } 410 | 411 | func (m *RpcSaslProto_SaslAuth) Reset() { *m = RpcSaslProto_SaslAuth{} } 412 | func (m *RpcSaslProto_SaslAuth) String() string { return proto.CompactTextString(m) } 413 | func (*RpcSaslProto_SaslAuth) ProtoMessage() {} 414 | 415 | func (m *RpcSaslProto_SaslAuth) GetMethod() string { 416 | if m != nil && m.Method != nil { 417 | return *m.Method 418 | } 419 | return "" 420 | } 421 | 422 | func (m *RpcSaslProto_SaslAuth) GetMechanism() string { 423 | if m != nil && m.Mechanism != nil { 424 | return *m.Mechanism 425 | } 426 | return "" 427 | } 428 | 429 | func (m *RpcSaslProto_SaslAuth) GetProtocol() string { 430 | if m != nil && m.Protocol != nil { 431 | return *m.Protocol 432 | } 433 | return "" 434 | } 435 | 436 | func (m *RpcSaslProto_SaslAuth) GetServerId() string { 437 | if m != nil && m.ServerId != nil { 438 | return *m.ServerId 439 | } 440 | return "" 441 | } 442 | 443 | func (m *RpcSaslProto_SaslAuth) GetChallenge() []byte { 444 | if m != nil { 445 | return m.Challenge 446 | } 447 | return nil 448 | } 449 | 450 | func init() { 451 | proto.RegisterEnum("hadoop.common.RpcKindProto", RpcKindProto_name, RpcKindProto_value) 452 | proto.RegisterEnum("hadoop.common.RpcRequestHeaderProto_OperationProto", RpcRequestHeaderProto_OperationProto_name, RpcRequestHeaderProto_OperationProto_value) 453 | proto.RegisterEnum("hadoop.common.RpcResponseHeaderProto_RpcStatusProto", RpcResponseHeaderProto_RpcStatusProto_name, RpcResponseHeaderProto_RpcStatusProto_value) 454 | proto.RegisterEnum("hadoop.common.RpcResponseHeaderProto_RpcErrorCodeProto", RpcResponseHeaderProto_RpcErrorCodeProto_name, RpcResponseHeaderProto_RpcErrorCodeProto_value) 455 | proto.RegisterEnum("hadoop.common.RpcSaslProto_SaslState", RpcSaslProto_SaslState_name, RpcSaslProto_SaslState_value) 456 | } 457 | -------------------------------------------------------------------------------- /go_hadoop_rpc/hadoop_common/Security.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. 2 | // source: Security.proto 3 | // DO NOT EDIT! 4 | 5 | package hadoop_common 6 | 7 | import proto "code.google.com/p/goprotobuf/proto" 8 | import json "encoding/json" 9 | import math "math" 10 | 11 | // Reference proto, json, and math imports to suppress error if they are not otherwise used. 12 | var _ = proto.Marshal 13 | var _ = &json.SyntaxError{} 14 | var _ = math.Inf 15 | 16 | // * 17 | // Security token identifier 18 | type TokenProto struct { 19 | Identifier []byte `protobuf:"bytes,1,req,name=identifier" json:"identifier,omitempty"` 20 | Password []byte `protobuf:"bytes,2,req,name=password" json:"password,omitempty"` 21 | Kind *string `protobuf:"bytes,3,req,name=kind" json:"kind,omitempty"` 22 | Service *string `protobuf:"bytes,4,req,name=service" json:"service,omitempty"` 23 | XXX_unrecognized []byte `json:"-"` 24 | } 25 | 26 | func (m *TokenProto) Reset() { *m = TokenProto{} } 27 | func (m *TokenProto) String() string { return proto.CompactTextString(m) } 28 | func (*TokenProto) ProtoMessage() {} 29 | 30 | func (m *TokenProto) GetIdentifier() []byte { 31 | if m != nil { 32 | return m.Identifier 33 | } 34 | return nil 35 | } 36 | 37 | func (m *TokenProto) GetPassword() []byte { 38 | if m != nil { 39 | return m.Password 40 | } 41 | return nil 42 | } 43 | 44 | func (m *TokenProto) GetKind() string { 45 | if m != nil && m.Kind != nil { 46 | return *m.Kind 47 | } 48 | return "" 49 | } 50 | 51 | func (m *TokenProto) GetService() string { 52 | if m != nil && m.Service != nil { 53 | return *m.Service 54 | } 55 | return "" 56 | } 57 | 58 | type GetDelegationTokenRequestProto struct { 59 | Renewer *string `protobuf:"bytes,1,req,name=renewer" json:"renewer,omitempty"` 60 | XXX_unrecognized []byte `json:"-"` 61 | } 62 | 63 | func (m *GetDelegationTokenRequestProto) Reset() { *m = GetDelegationTokenRequestProto{} } 64 | func (m *GetDelegationTokenRequestProto) String() string { return proto.CompactTextString(m) } 65 | func (*GetDelegationTokenRequestProto) ProtoMessage() {} 66 | 67 | func (m *GetDelegationTokenRequestProto) GetRenewer() string { 68 | if m != nil && m.Renewer != nil { 69 | return *m.Renewer 70 | } 71 | return "" 72 | } 73 | 74 | type GetDelegationTokenResponseProto struct { 75 | Token *TokenProto `protobuf:"bytes,1,opt,name=token" json:"token,omitempty"` 76 | XXX_unrecognized []byte `json:"-"` 77 | } 78 | 79 | func (m *GetDelegationTokenResponseProto) Reset() { *m = GetDelegationTokenResponseProto{} } 80 | func (m *GetDelegationTokenResponseProto) String() string { return proto.CompactTextString(m) } 81 | func (*GetDelegationTokenResponseProto) ProtoMessage() {} 82 | 83 | func (m *GetDelegationTokenResponseProto) GetToken() *TokenProto { 84 | if m != nil { 85 | return m.Token 86 | } 87 | return nil 88 | } 89 | 90 | type RenewDelegationTokenRequestProto struct { 91 | Token *TokenProto `protobuf:"bytes,1,req,name=token" json:"token,omitempty"` 92 | XXX_unrecognized []byte `json:"-"` 93 | } 94 | 95 | func (m *RenewDelegationTokenRequestProto) Reset() { *m = RenewDelegationTokenRequestProto{} } 96 | func (m *RenewDelegationTokenRequestProto) String() string { return proto.CompactTextString(m) } 97 | func (*RenewDelegationTokenRequestProto) ProtoMessage() {} 98 | 99 | func (m *RenewDelegationTokenRequestProto) GetToken() *TokenProto { 100 | if m != nil { 101 | return m.Token 102 | } 103 | return nil 104 | } 105 | 106 | type RenewDelegationTokenResponseProto struct { 107 | NewExpiryTime *uint64 `protobuf:"varint,1,req,name=newExpiryTime" json:"newExpiryTime,omitempty"` 108 | XXX_unrecognized []byte `json:"-"` 109 | } 110 | 111 | func (m *RenewDelegationTokenResponseProto) Reset() { *m = RenewDelegationTokenResponseProto{} } 112 | func (m *RenewDelegationTokenResponseProto) String() string { return proto.CompactTextString(m) } 113 | func (*RenewDelegationTokenResponseProto) ProtoMessage() {} 114 | 115 | func (m *RenewDelegationTokenResponseProto) GetNewExpiryTime() uint64 { 116 | if m != nil && m.NewExpiryTime != nil { 117 | return *m.NewExpiryTime 118 | } 119 | return 0 120 | } 121 | 122 | type CancelDelegationTokenRequestProto struct { 123 | Token *TokenProto `protobuf:"bytes,1,req,name=token" json:"token,omitempty"` 124 | XXX_unrecognized []byte `json:"-"` 125 | } 126 | 127 | func (m *CancelDelegationTokenRequestProto) Reset() { *m = CancelDelegationTokenRequestProto{} } 128 | func (m *CancelDelegationTokenRequestProto) String() string { return proto.CompactTextString(m) } 129 | func (*CancelDelegationTokenRequestProto) ProtoMessage() {} 130 | 131 | func (m *CancelDelegationTokenRequestProto) GetToken() *TokenProto { 132 | if m != nil { 133 | return m.Token 134 | } 135 | return nil 136 | } 137 | 138 | type CancelDelegationTokenResponseProto struct { 139 | XXX_unrecognized []byte `json:"-"` 140 | } 141 | 142 | func (m *CancelDelegationTokenResponseProto) Reset() { *m = CancelDelegationTokenResponseProto{} } 143 | func (m *CancelDelegationTokenResponseProto) String() string { return proto.CompactTextString(m) } 144 | func (*CancelDelegationTokenResponseProto) ProtoMessage() {} 145 | 146 | func init() { 147 | } 148 | -------------------------------------------------------------------------------- /go_hadoop_rpc/hadoop_common/ZKFCProtocol.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. 2 | // source: ZKFCProtocol.proto 3 | // DO NOT EDIT! 4 | 5 | package hadoop_common 6 | 7 | import proto "code.google.com/p/goprotobuf/proto" 8 | import json "encoding/json" 9 | import math "math" 10 | 11 | // Reference proto, json, and math imports to suppress error if they are not otherwise used. 12 | var _ = proto.Marshal 13 | var _ = &json.SyntaxError{} 14 | var _ = math.Inf 15 | 16 | type CedeActiveRequestProto struct { 17 | MillisToCede *uint32 `protobuf:"varint,1,req,name=millisToCede" json:"millisToCede,omitempty"` 18 | XXX_unrecognized []byte `json:"-"` 19 | } 20 | 21 | func (m *CedeActiveRequestProto) Reset() { *m = CedeActiveRequestProto{} } 22 | func (m *CedeActiveRequestProto) String() string { return proto.CompactTextString(m) } 23 | func (*CedeActiveRequestProto) ProtoMessage() {} 24 | 25 | func (m *CedeActiveRequestProto) GetMillisToCede() uint32 { 26 | if m != nil && m.MillisToCede != nil { 27 | return *m.MillisToCede 28 | } 29 | return 0 30 | } 31 | 32 | type CedeActiveResponseProto struct { 33 | XXX_unrecognized []byte `json:"-"` 34 | } 35 | 36 | func (m *CedeActiveResponseProto) Reset() { *m = CedeActiveResponseProto{} } 37 | func (m *CedeActiveResponseProto) String() string { return proto.CompactTextString(m) } 38 | func (*CedeActiveResponseProto) ProtoMessage() {} 39 | 40 | type GracefulFailoverRequestProto struct { 41 | XXX_unrecognized []byte `json:"-"` 42 | } 43 | 44 | func (m *GracefulFailoverRequestProto) Reset() { *m = GracefulFailoverRequestProto{} } 45 | func (m *GracefulFailoverRequestProto) String() string { return proto.CompactTextString(m) } 46 | func (*GracefulFailoverRequestProto) ProtoMessage() {} 47 | 48 | type GracefulFailoverResponseProto struct { 49 | XXX_unrecognized []byte `json:"-"` 50 | } 51 | 52 | func (m *GracefulFailoverResponseProto) Reset() { *m = GracefulFailoverResponseProto{} } 53 | func (m *GracefulFailoverResponseProto) String() string { return proto.CompactTextString(m) } 54 | func (*GracefulFailoverResponseProto) ProtoMessage() {} 55 | 56 | func init() { 57 | } 58 | -------------------------------------------------------------------------------- /go_hadoop_rpc/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "errors" 7 | "flag" 8 | "io" 9 | "log" 10 | "net" 11 | 12 | "./hadoop_common" 13 | ) 14 | 15 | func panicOnErr(err error) { 16 | if err != nil { 17 | panic(err) 18 | } 19 | } 20 | 21 | type AuthMethod byte 22 | 23 | const ( 24 | SIMPLE AuthMethod = 80 + iota 25 | KEREBROS 26 | DIGEST 27 | PLAIN 28 | ) 29 | 30 | type AuthProtocol byte 31 | 32 | const ( 33 | NONE AuthProtocol = 0 34 | SASL = int8(-33) 35 | ) 36 | 37 | type RpcHeader struct { 38 | Header [4]byte 39 | Version byte 40 | ServiceClass byte 41 | AuthProtocol AuthProtocol 42 | } 43 | 44 | type RpcPrequel struct { 45 | Header RpcHeader 46 | ReqHeader hadoop_common.RpcRequestHeaderProto 47 | ConnContext hadoop_common.IpcConnectionContextProto 48 | } 49 | 50 | func ReadRpcPrequel(r io.Reader) (preq *RpcPrequel, err error) { 51 | preq = new(RpcPrequel) 52 | preq.Header, err = ReadRpcHeader(r) 53 | readint32(r) 54 | if err != nil { 55 | return nil, err 56 | } 57 | if err := readProto(r, &preq.ReqHeader); err != nil { 58 | return nil, err 59 | } 60 | if err := readProto(r, &preq.ConnContext); err != nil { 61 | return nil, err 62 | } 63 | return preq, nil 64 | } 65 | 66 | func ReadRpcHeader(r io.Reader) (RpcHeader, error) { 67 | var header RpcHeader 68 | err := binary.Read(r, binary.BigEndian, &header) 69 | return header, err 70 | } 71 | 72 | func server(addr string) { 73 | l, err := net.Listen("tcp", addr) 74 | if err != nil { 75 | log.Fatalln("cannot listen to", addr, err) 76 | } 77 | for { 78 | c, err := l.Accept() 79 | if err != nil { 80 | log.Println("cannot accept connection", err) 81 | continue 82 | } 83 | go func() { 84 | defer c.Close() 85 | if _, err := ReadRpcPrequel(c); err != nil { 86 | log.Println("Cannot read prequel", err) 87 | return 88 | } 89 | for { 90 | size, err := readint32(c) 91 | payload := make([]byte, size) 92 | _, err = io.ReadFull(c, payload) 93 | 94 | r := bytes.NewReader(payload) 95 | var rpcheader hadoop_common.RpcRequestHeaderProto 96 | readProto(r, &rpcheader) 97 | rpcVersion, err := readint64(r) 98 | declaringClassProtocolName, err := readString(r) 99 | methodName, err := readString(r) 100 | clientVersion, err := readint64(r) 101 | clientMethodHash, err := readint32(r) 102 | parameterClassesLength, err := readint32(r) 103 | if err != nil { 104 | log.Println("error parsing client request", err) 105 | return 106 | } 107 | log.Println("rpc version", rpcVersion, "client version", clientVersion, "hash", clientMethodHash) 108 | log.Printf("%s: %s(%d args...)", declaringClassProtocolName, methodName, parameterClassesLength) 109 | } 110 | }() 111 | } 112 | } 113 | 114 | func main() { 115 | saddr := flag.String("server", "", "address to listen for RPC requests") 116 | flag.Parse() 117 | if *saddr != "" { 118 | server(*saddr) 119 | } 120 | } 121 | 122 | func writeVarint32(value int, w io.Writer) error { 123 | for { 124 | if (value &^ 0x7F) == 0 { 125 | if _, err := w.Write([]byte{byte(value)}); err != nil { 126 | return err 127 | } 128 | return nil 129 | } else { 130 | if _, err := w.Write([]byte{byte((value & 0x7F) | 0x80)}); err != nil { 131 | return err 132 | } 133 | value >>= 7; 134 | } 135 | } 136 | panic("unreachable") 137 | } 138 | 139 | func readByte(r io.Reader, perr *error) byte { 140 | if *perr != nil { 141 | return 0 142 | } 143 | var b [1]byte 144 | if _, *perr = r.Read(b[:]); *perr != nil { 145 | return 0 146 | } 147 | return b[0] 148 | } 149 | 150 | func readVarint32(r io.Reader) (int32, error) { 151 | var err error 152 | b := readByte(r, &err) 153 | if b & 0x80 == 0 { 154 | return int32(b), err 155 | } 156 | result := int32(b & 0x7F) 157 | if tmp := readByte(r, &err); tmp & 0x80 == 0 { 158 | return result | int32(tmp) << 7, err 159 | } else { 160 | result |= (int32(tmp) & 0x7F) << 7 161 | } 162 | if tmp := readByte(r, &err); tmp & 0x80 == 0 { 163 | return result | int32(tmp) << 14, err 164 | } else { 165 | result |= (int32(tmp) & 0x7F) << 14 166 | } 167 | if tmp := readByte(r, &err); tmp & 0x80 == 0 { 168 | return result | int32(tmp) << 21, err 169 | } else { 170 | result |= (int32(tmp) & 0x7F) << 21 171 | } 172 | tmp := readByte(r, &err) 173 | result |= int32(tmp) << 28 174 | if tmp & 0x80 != 0 { 175 | for i := 0; i < 5; i++ { 176 | if readByte(r, &err) & 0x80 == 0 { 177 | return result, err 178 | } 179 | } 180 | return 0, errors.New("malformed varint32") 181 | } 182 | return result, err 183 | } 184 | 185 | -------------------------------------------------------------------------------- /go_hadoop_rpc/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "code.google.com/p/goprotobuf/proto" 6 | "encoding/binary" 7 | "fmt" 8 | "io" 9 | "testing" 10 | 11 | "./hadoop_common" 12 | "launchpad.net/gocheck" 13 | ) 14 | 15 | var clientInput = []byte{ 16 | //h r p c 17 | 0x68, 0x72, 0x70, 0x63, 18 | // version, service class, AuthProtocol 19 | 0x09, 0x00, 0x00, 20 | // size of next two size delimited protobuf objets: 21 | // RpcRequestHeader and IpcConnectionContext 22 | 0x00, 0x00, 0x00, 0x32, // = 50 23 | // varint encoding of RpcRequestHeader length 24 | 0x1e, // = 30 25 | 0x08, 0x02, 0x10, 0x00, 0x18, 0xfd, 0xff, 0xff, 0xff, 0x0f, 26 | 0x22, 0x10, 0x87, 0xeb, 0x86, 0xd4, 0x9c, 0x95, 0x4c, 0x15, 27 | 0x8a, 0xb0, 0xd7, 0xbc, 0x2e, 0xca, 0xca, 0x37, 0x28, 0x01, 28 | // varint encoding of IpcConnectionContext length 29 | 0x12, // = 18 30 | 0x12, 0x0a, 0x0a, 0x08, 0x65, 0x6c, 0x65, 0x69, 0x62, 0x6f, 31 | 0x76, 0x69, 0x1a, 0x04, 0x70, 0x69, 0x6e, 0x67, 32 | // Size of RpcRequestHeader + RpcRequest protobuf objects 33 | 0x00, 0x00, 0x00, 0x3f, // = 63 34 | // varint size of RpcRequest Header 35 | 0x1a, // = 26 36 | 0x08, 0x01, 0x10, 0x00, 0x18, 0x00, 0x22, 0x10, 0x87, 0xeb, 37 | 0x86, 0xd4, 0x9c, 0x95, 0x4c, 0x15, 0x8a, 0xb0, 0xd7, 0xbc, 38 | 0x2e, 0xca, 0xca, 0x37, 0x28, 0x00, 39 | // RPC Request writable. It's not size delimited 40 | // long - RPC version 41 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, // = 2 42 | // utf8 string - protocol name 43 | 0x00, 0x04, // string legnth = 4 44 | // p i n g 45 | 0x70, 0x69, 0x6e, 0x67, 46 | // utf8 string - method name 47 | 0x00, 0x04, // string legnth = 4 48 | // p i n g 49 | 0x70, 0x69, 0x6e, 0x67, 50 | // long - client version 51 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, // = 1 52 | // int - client method hash 53 | 0xa0, 0xbd, 0x17, 0xcc, 54 | // int - parameter class length 55 | 0x00, 0x00, 0x00, 0x00, 56 | // ping request 57 | 0xff, 0xff, 0xff, 0xff, 58 | } 59 | 60 | var serverInput = []byte{ 61 | // size of entire request 62 | 0x00, 0x00, 0x00, 0x33, 63 | // varint size of RpcResponseHeader 64 | 0x1a, // 16 + 10 = 26 65 | 0x08, 0x00, 0x10, 0x00, 0x18, 0x09, 0x3a, 0x10, 0x9b, 0x19, 66 | 0x9b, 0x41, 0x4d, 0x86, 0x42, 0xd7, 0x94, 0x79, 0x3f, 0x4b, 67 | 0x16, 0xa0, 0x22, 0x7c, 0x40, 0x00, 68 | // Writable response 69 | // short - length of declared class 70 | 0x00, 0x10, 71 | // j a v a . l a n g . 72 | 0x6a, 0x61, 0x76, 0x61, 0x2e, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 73 | // S t r i n g 74 | 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 75 | // short - length of value 76 | 0x00, 0x04, 77 | // p o n g 78 | 0x70, 0x6f, 0x6e, 0x67, 79 | } 80 | 81 | func Test(t *testing.T) { gocheck.TestingT(t) } 82 | 83 | type MySuite struct{} 84 | 85 | var _ = gocheck.Suite(&MySuite{}) 86 | 87 | func readPayload(payload []byte, c *gocheck.C) { 88 | r := bytes.NewReader(payload) 89 | var rpcheader hadoop_common.RpcRequestHeaderProto 90 | panicOnErr(readProto(r, &rpcheader)) 91 | fmt.Printf("%+#v", rpcheader) 92 | 93 | rpcVersion, err := readint64(r) 94 | panicOnErr(err) 95 | c.Check(rpcVersion, gocheck.Equals, int64(2)) 96 | declaringClassProtocolName, err := readString(r) 97 | panicOnErr(err) 98 | c.Check(declaringClassProtocolName, gocheck.Equals, "ping") 99 | methodName, err := readString(r) 100 | panicOnErr(err) 101 | c.Check(methodName, gocheck.Equals, "ping") 102 | clientVersion, err := readint64(r) 103 | panicOnErr(err) 104 | c.Check(clientVersion, gocheck.Equals, int64(1)) 105 | clientMethodHash, err := readint32(r) 106 | panicOnErr(err) 107 | fmt.Println("clientMethodHash:", clientMethodHash) 108 | parameterClassesLength, err := readint32(r) 109 | panicOnErr(err) 110 | fmt.Println("parameterClassesLength:", parameterClassesLength) 111 | } 112 | 113 | func (s *MySuite) TestClientInfo(c *gocheck.C) { 114 | r := bytes.NewReader(clientInput) 115 | preq, err := ReadRpcPrequel(r) 116 | panicOnErr(err) 117 | h := preq.Header 118 | c.Assert(string(h.Header[:]), gocheck.Equals, "hrpc") 119 | c.Check(h.Version >= 8, gocheck.Equals, true) 120 | c.Check(h.AuthProtocol, gocheck.Equals, NONE) 121 | c.Assert(h.ServiceClass, gocheck.Equals, byte(0)) 122 | 123 | var payload_size int32 124 | panicOnErr(binary.Read(r, binary.BigEndian, &payload_size)) 125 | fmt.Println("payload size: ", payload_size) 126 | payload := make([]byte, payload_size) 127 | _, err = io.ReadFull(r, payload) 128 | panicOnErr(err) 129 | readPayload(payload, c) 130 | } 131 | -------------------------------------------------------------------------------- /go_hadoop_rpc/proto_helper.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "code.google.com/p/goprotobuf/proto" 5 | "io" 6 | ) 7 | 8 | func readProtoBytes(r io.Reader) ([]byte, error) { 9 | size, err := readVarint32(r) 10 | if err != nil { 11 | return nil, err 12 | } 13 | msg := make([]byte, size) 14 | _, err = io.ReadFull(r, msg) 15 | return msg, err 16 | } 17 | 18 | func readProto(r io.Reader, pmsg proto.Message) error { 19 | msg, err := readProtoBytes(r) 20 | if err != nil { 21 | return err 22 | } 23 | return proto.Unmarshal(msg, pmsg) 24 | } 25 | -------------------------------------------------------------------------------- /go_hadoop_rpc/server.dat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elazarl/hadoop_rpc_walktrhough/1d9aabef04499aa7226d730d9411d8d898567c64/go_hadoop_rpc/server.dat -------------------------------------------------------------------------------- /go_hadoop_rpc/writables.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/binary" 5 | "io" 6 | ) 7 | 8 | func readint16(r io.Reader) (int16, error) { 9 | var d int16 10 | err := binary.Read(r, binary.BigEndian, &d) 11 | return d, err 12 | } 13 | 14 | func readint32(r io.Reader) (int32, error) { 15 | var d int32 16 | err := binary.Read(r, binary.BigEndian, &d) 17 | return d, err 18 | } 19 | 20 | func readint64(r io.Reader) (int64, error) { 21 | var d int64 22 | err := binary.Read(r, binary.BigEndian, &d) 23 | return d, err 24 | } 25 | 26 | func readString(r io.Reader) (string, error) { 27 | size, err := readint16(r) 28 | if err != nil { 29 | return "", err 30 | } 31 | s := make([]byte, size) 32 | if _, err := io.ReadFull(r, s); err != nil { 33 | return "", err 34 | } 35 | return string(s), nil 36 | } 37 | -------------------------------------------------------------------------------- /lib/org/apache/hadoop/hadoop-minikdc/3.0.0-SNAPSHOT/hadoop-minikdc-3.0.0-SNAPSHOT-sources.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elazarl/hadoop_rpc_walktrhough/1d9aabef04499aa7226d730d9411d8d898567c64/lib/org/apache/hadoop/hadoop-minikdc/3.0.0-SNAPSHOT/hadoop-minikdc-3.0.0-SNAPSHOT-sources.jar -------------------------------------------------------------------------------- /lib/org/apache/hadoop/hadoop-minikdc/3.0.0-SNAPSHOT/hadoop-minikdc-3.0.0-SNAPSHOT.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/elazarl/hadoop_rpc_walktrhough/1d9aabef04499aa7226d730d9411d8d898567c64/lib/org/apache/hadoop/hadoop-minikdc/3.0.0-SNAPSHOT/hadoop-minikdc-3.0.0-SNAPSHOT.jar -------------------------------------------------------------------------------- /lib/org/apache/hadoop/hadoop-minikdc/3.0.0-SNAPSHOT/hadoop-minikdc-3.0.0-SNAPSHOT.pom: -------------------------------------------------------------------------------- 1 | 2 | 15 | 18 | 19 | org.apache.hadoop 20 | hadoop-project 21 | 3.0.0-SNAPSHOT 22 | ../../hadoop-project 23 | 24 | 4.0.0 25 | hadoop-minikdc 26 | 3.0.0-SNAPSHOT 27 | Apache Hadoop MiniKDC 28 | Apache Hadoop MiniKDC 29 | jar 30 | 31 | 32 | 33 | commons-io 34 | commons-io 35 | compile 36 | 37 | 38 | org.apache.directory.server 39 | apacheds-all 40 | 2.0.0-M15 41 | compile 42 | 43 | 44 | org.slf4j 45 | slf4j-log4j12 46 | compile 47 | 48 | 49 | junit 50 | junit 51 | compile 52 | 53 | 54 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | com.github.elazar 6 | hadoop_rpc 7 | 1.0-SNAPSHOT 8 | jar 9 | 10 | hadoop_rpc 11 | http://github.com/elazarl/hadoop_rpc 12 | 13 | 14 | UTF-8 15 | 16 | 17 | 18 | 19 | apache-repository 20 | Apache Repository 21 | https://repository.apache.org/service/local/repositories/orgapachehadoop-099/content/ 22 | 23 | 27 | 28 | local 29 | file://${basedir}/lib 30 | 31 | 32 | 33 | 34 | 35 | org.apache.hadoop 36 | hadoop-client 37 | 2.1.0-beta 38 | 39 | 40 | org.apache.hadoop 41 | hadoop-hdfs 42 | 2.1.0-beta 43 | 44 | 45 | 46 | org.apache.hadoop 47 | hadoop-minikdc 48 | 3.0.0-SNAPSHOT 49 | test 50 | 51 | 52 | 53 | 54 | junit 55 | junit 56 | 4.11 57 | test 58 | 59 | 60 | org.hamcrest 61 | hamcrest-all 62 | 1.3 63 | test 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /src/main/java/com/github/elazar/hadoop/examples/HadoopBasicUsage.java: -------------------------------------------------------------------------------- 1 | package com.github.elazar.hadoop.examples; 2 | 3 | import com.google.common.base.Charsets; 4 | import org.apache.hadoop.conf.Configured; 5 | import org.apache.hadoop.fs.*; 6 | import org.apache.hadoop.security.UserGroupInformation; 7 | import org.apache.hadoop.util.Tool; 8 | import org.apache.hadoop.util.ToolRunner; 9 | 10 | import java.security.PrivilegedExceptionAction; 11 | import java.util.Arrays; 12 | 13 | /** 14 | * Show how Hadoop program sets its configuration, and thus get its user. 15 | */ 16 | public class HadoopBasicUsage extends Configured implements Tool { 17 | public static void main(String[] args) throws Exception { 18 | System.out.println("args: " + Arrays.toString(args)); 19 | System.exit(ToolRunner.run(new HadoopBasicUsage(), args)); 20 | } 21 | 22 | @Override 23 | public int run(String[] args) throws Exception { 24 | final org.apache.hadoop.conf.Configuration conf = getConf(); 25 | 26 | final String cwd = System.getProperty("user.dir"); 27 | System.setProperty("java.security.krb5.conf", cwd + "/krb5.conf"); 28 | System.setProperty("sun.security.krb5.debug", "true"); 29 | 30 | UserGroupInformation.setConfiguration(conf); 31 | UserGroupInformation ugi = UserGroupInformation. 32 | loginUserFromKeytabAndReturnUGI("hdfs/EXAMPLE.COM", cwd + "/keytab"); 33 | 34 | 35 | ugi.doAs(new PrivilegedExceptionAction() { 36 | @Override 37 | public Object run() throws Exception { 38 | System.out.println(">>" + UserGroupInformation.getCurrentUser().getAuthenticationMethod()); 39 | final FileSystem fs = FileSystem.get(conf); 40 | int i=0; 41 | final FSDataOutputStream out = fs.create(new Path("/foo_" + System.currentTimeMillis())); 42 | out.write("foo".getBytes(Charsets.US_ASCII)); 43 | out.close(); 44 | 45 | for (FileStatus status : fs.listStatus(new Path("/"))) { 46 | System.out.println(status.getPath()); 47 | System.out.println(status); 48 | if (i++ > 10) { 49 | System.out.println("only first 10 showed..."); 50 | break; 51 | } 52 | } 53 | return null; 54 | } 55 | }); 56 | 57 | return 0; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/github/elazar/hadoop/examples/HadoopRPC.java: -------------------------------------------------------------------------------- 1 | package com.github.elazar.hadoop.examples; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.ipc.ProtocolInfo; 5 | import org.apache.hadoop.ipc.RPC; 6 | 7 | import java.io.IOException; 8 | import java.net.InetSocketAddress; 9 | 10 | /** 11 | * Hello world! 12 | * 13 | */ 14 | public class HadoopRPC 15 | { 16 | @ProtocolInfo(protocolName = "ping", protocolVersion = 1) 17 | public static interface PingProtocol { 18 | String ping(); 19 | } 20 | 21 | public static class Ping implements PingProtocol { 22 | public String ping() { 23 | System.out.println("Server: "); 24 | return "pong"; 25 | } 26 | } 27 | 28 | public static InetSocketAddress addr = new InetSocketAddress("localhost", 5121); 29 | 30 | public static RPC.Server server() throws IOException { 31 | final RPC.Server server = new RPC.Builder(new Configuration()). 32 | setBindAddress(addr.getHostName()). 33 | setPort(addr.getPort()). 34 | setInstance(new Ping()). 35 | setProtocol(PingProtocol.class). 36 | build(); 37 | server.start(); 38 | return server; 39 | } 40 | 41 | public static void client() throws IOException { 42 | final PingProtocol proxy = RPC.getProxy(PingProtocol.class, RPC.getProtocolVersion(PingProtocol.class), 43 | addr, new Configuration()); 44 | System.out.println("Client: ping " + proxy.ping()); 45 | } 46 | 47 | public static void main(String[] args ) throws IOException { 48 | final String runThis = args.length > 0 ? args[0] : ""; 49 | if (runThis.equals("server")) { 50 | server(); 51 | } else if (runThis.equals("client")) { 52 | client(); 53 | } else { 54 | final RPC.Server server = server(); 55 | client(); 56 | server.stop(); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/com/github/elazar/hadoop/examples/TheProtocolProxy.java: -------------------------------------------------------------------------------- 1 | package com.github.elazar.hadoop.examples; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.ipc.*; 5 | 6 | import java.io.IOException; 7 | import java.net.InetSocketAddress; 8 | import java.util.concurrent.ExecutorService; 9 | import java.util.concurrent.Executors; 10 | 11 | /** 12 | * An example of proper usage of Hadoop RPC 13 | */ 14 | public class TheProtocolProxy { 15 | @ProtocolInfo(protocolName = "ping", protocolVersion = 42) 16 | public static interface PingProtocol extends VersionedProtocol { 17 | 18 | public abstract String ping(); 19 | } 20 | 21 | public static class Ping implements PingProtocol { 22 | @Override 23 | public long getProtocolVersion(String protocol, long clientVersion) throws IOException { 24 | return RPC.getProtocolVersion(PingProtocol.class); 25 | } 26 | 27 | @Override 28 | public ProtocolSignature getProtocolSignature(String protocol, long clientVersion, int clientMethodsHash) throws IOException { 29 | return ProtocolSignature.getProtocolSignature(clientMethodsHash, 30 | RPC.getProtocolVersion(PingProtocol.class), 31 | PingProtocol.class); 32 | } 33 | 34 | public String ping() { 35 | return "pong"; 36 | } 37 | } 38 | public static void main( String[] args ) throws IOException { 39 | final RPC.Server server = new RPC.Builder(new Configuration()). 40 | setInstance(new Ping()). 41 | setProtocol(PingProtocol.class). 42 | build(); 43 | ExecutorService svc = Executors.newCachedThreadPool(); 44 | // server.start(); 45 | 46 | final ProtocolProxy protocolProxy = RPC.getProtocolProxy(PingProtocol.class, 47 | RPC.getProtocolVersion(PingProtocol.class), 48 | server.getListenerAddress(), new Configuration()); 49 | System.out.println("method: " + protocolProxy.isMethodSupported("ping")); 50 | 51 | final PingProtocol proxy = RPC.getProxy(PingProtocol.class, RPC.getProtocolVersion(PingProtocol.class), 52 | server.getListenerAddress(), new Configuration()); 53 | System.out.println("Server: ping " + proxy.ping()); 54 | System.out.println( "Hello World!" ); 55 | server.stop(); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO, stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n -------------------------------------------------------------------------------- /src/test/java/com/github/elazar/hadoop/examples/SaslGssapiExampleTest.java: -------------------------------------------------------------------------------- 1 | package com.github.elazar.hadoop.examples; 2 | 3 | import com.google.common.collect.ImmutableMap; 4 | import org.apache.hadoop.minikdc.MiniKdc; 5 | import org.junit.Ignore; 6 | import org.junit.Test; 7 | 8 | import javax.security.auth.Subject; 9 | import javax.security.auth.callback.*; 10 | import javax.security.auth.login.AppConfigurationEntry; 11 | import javax.security.auth.login.Configuration; 12 | import javax.security.auth.login.LoginContext; 13 | import javax.security.sasl.*; 14 | import java.io.File; 15 | import java.io.IOException; 16 | import java.security.PrivilegedAction; 17 | import java.util.Arrays; 18 | 19 | import static org.hamcrest.Matchers.is; 20 | import static org.hamcrest.Matchers.lessThanOrEqualTo; 21 | import static org.junit.Assert.assertThat; 22 | 23 | /** 24 | * Show simple use of GSSAPI SASL authentication 25 | */ 26 | @Ignore 27 | public class SaslGssapiExampleTest { 28 | 29 | @Test 30 | public void testSaslGssapi() throws Exception { 31 | final MiniKdc kdc = makeKdc(); 32 | kdc.start(); 33 | 34 | try { 35 | Subject serverSubject = getServerKdcSubject(kdc); 36 | Subject clientSubject = SubjectgetClientKdcSubject(kdc); 37 | 38 | final SaslClient client = Subject.doAs(clientSubject, new PrivilegedAction() { 39 | @Override 40 | public SaslClient run() { 41 | try { 42 | return Sasl.createSaslClient( 43 | new String[]{"GSSAPI"}, 44 | // authzId is not needed in general, 45 | // see http://www.openldap.org/lists/openldap-devel/200011/msg00036.html 46 | null, 47 | "server", kdc.getRealm(), 48 | null, null 49 | ); 50 | } catch (SaslException e) { 51 | throw new RuntimeException(e); 52 | } 53 | } 54 | }); 55 | SaslServer server = Subject.doAs(serverSubject, new PrivilegedAction() { 56 | @Override 57 | public SaslServer run() { 58 | try { 59 | return Sasl.createSaslServer("GSSAPI", "server", kdc.getRealm(), null, new CallbackHandler() { 60 | @Override 61 | public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException { 62 | for (Callback callback : callbacks) { 63 | if (callback instanceof AuthorizeCallback) { 64 | final AuthorizeCallback authorizeCb = (AuthorizeCallback) callback; 65 | final String user = 66 | authorizeCb.getAuthenticationID(); 67 | if (user.equals("client@" + kdc.getRealm())) { 68 | authorizeCb.setAuthorized(true); 69 | } else { 70 | authorizeCb.setAuthorized(false); 71 | } 72 | } 73 | } 74 | } 75 | }); 76 | } catch (SaslException e) { 77 | throw new RuntimeException(e); 78 | } 79 | } 80 | }); 81 | // We know that in GSSAPI client has initial response 82 | assertThat(client.hasInitialResponse(), is(true)); 83 | byte[] resp = new byte[0]; 84 | int nclient = 0, nserver = 0; 85 | while (!client.isComplete() && !server.isComplete()) { 86 | if (!client.isComplete()) { 87 | nclient++; 88 | resp = evaluateChallengeAs(clientSubject, client, resp); 89 | } 90 | if (!server.isComplete()) { 91 | nserver++; 92 | resp = evaluateResponseAs(serverSubject, server, resp); 93 | } 94 | // cast to long since Math.abs(Integer.MIN_VALUE) < 0. 95 | assertThat(Math.abs((long)(nclient-nserver)), lessThanOrEqualTo(1L)); 96 | } 97 | assertThat(server.isComplete(), is(true)); 98 | assertThat(client.isComplete(), is(true)); 99 | } finally { 100 | kdc.stop(); 101 | } 102 | } 103 | 104 | private byte[] evaluateResponseAs(final Subject serverSubject, final SaslServer server, final byte[] challenge) { 105 | return (byte[]) Subject.doAs(serverSubject, new PrivilegedAction() { 106 | @Override 107 | public Object run() { 108 | try { 109 | return server.evaluateResponse(challenge); 110 | } catch (SaslException e) { 111 | throw new RuntimeException(e); 112 | } 113 | } 114 | }); 115 | } 116 | 117 | private byte[] evaluateChallengeAs(Subject clientSubject, final SaslClient client, final byte[] challenge) { 118 | return (byte[]) Subject.doAs(clientSubject, new PrivilegedAction() { 119 | @Override 120 | public Object run() { 121 | try { 122 | return client.evaluateChallenge(challenge); 123 | } catch (SaslException e) { 124 | throw new RuntimeException(e); 125 | } 126 | } 127 | }); 128 | } 129 | 130 | private Subject SubjectgetClientKdcSubject(final MiniKdc kdc) throws Exception { 131 | final String principal = "client"; 132 | final String password = "secret pass"; 133 | kdc.createPrincipal(principal, password); 134 | 135 | Configuration conf = new Configuration() { 136 | @Override 137 | public AppConfigurationEntry[] getAppConfigurationEntry(String name) { 138 | return new AppConfigurationEntry[] { 139 | new AppConfigurationEntry(getKrb5LoginModuleName(), 140 | AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, 141 | ImmutableMap.builder(). 142 | put("debug", "true"). 143 | put("principal", principal + "@" + kdc.getRealm()). 144 | put("useKeyTab", "false"). 145 | build()) 146 | }; 147 | } 148 | }; 149 | LoginContext login = new LoginContext("", new Subject(), 150 | new CallbackHandler() { 151 | @Override 152 | public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException { 153 | System.out.println(Arrays.toString(callbacks)); 154 | for (Callback callback : callbacks) { 155 | if (callback instanceof PasswordCallback) { 156 | ((PasswordCallback) callback).setPassword(password.toCharArray()); 157 | } else if (callback instanceof NameCallback) { 158 | ((NameCallback) callback).setName("client@" + kdc.getRealm()); 159 | } 160 | } 161 | } 162 | }, 163 | conf); 164 | login.login(); 165 | return login.getSubject(); 166 | } 167 | 168 | private Subject getServerKdcSubject(final MiniKdc kdc) throws Exception { 169 | // for some reason, Java appends "/realm" to the principal name given to it 170 | // in the createSasl* methods. You can probably override this behaviour by 171 | // setting an option to the SaslClient or to the JAAS login module, but I'm 172 | // too exhausted figuring out how to do that. 173 | final String principal = "server/" + kdc.getRealm().toLowerCase(); 174 | final File keyTab = new File(getWorkingDir(), "keyTab"); 175 | kdc.createPrincipal(keyTab, principal); 176 | 177 | Configuration conf = new Configuration() { 178 | @Override 179 | public AppConfigurationEntry[] getAppConfigurationEntry(String name) { 180 | return new AppConfigurationEntry[] { 181 | new AppConfigurationEntry(getKrb5LoginModuleName(), 182 | AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, 183 | ImmutableMap.builder(). 184 | put("debug", "true"). 185 | put("principal", principal + "@" + kdc.getRealm()). 186 | put("useKeyTab", "true"). 187 | put("keyTab", keyTab.getAbsolutePath()). 188 | put("storeKey", "true"). 189 | build()) 190 | }; 191 | } 192 | }; 193 | LoginContext login = new LoginContext("", new Subject(), null, conf); 194 | login.login(); 195 | return login.getSubject(); 196 | } 197 | 198 | public static String getKrb5LoginModuleName() { 199 | return System.getProperty("java.vendor").contains("IBM") 200 | ? "com.ibm.security.auth.module.Krb5LoginModule" 201 | : "com.sun.security.auth.module.Krb5LoginModule"; 202 | } 203 | 204 | private MiniKdc makeKdc() throws Exception { 205 | return new MiniKdc(MiniKdc.createConf(), getWorkingDir()); 206 | } 207 | 208 | private File getWorkingDir() { 209 | return new File("target"); 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /src/test/java/com/github/elazar/hadoop/examples/SaslMd5DigestExampleTest.java: -------------------------------------------------------------------------------- 1 | package com.github.elazar.hadoop.examples; 2 | 3 | import com.google.common.base.Charsets; 4 | import com.google.common.collect.ImmutableMap; 5 | import org.junit.Test; 6 | 7 | import javax.security.auth.callback.*; 8 | import javax.security.sasl.*; 9 | import java.io.IOException; 10 | import java.util.Map; 11 | 12 | import static org.hamcrest.Matchers.*; 13 | import static org.junit.Assert.assertThat; 14 | 15 | /** 16 | * Show example Sasl Usage 17 | */ 18 | public class SaslMd5DigestExampleTest { 19 | 20 | final String[] digestMd5 = {"DIGEST-MD5"}; 21 | final Map qopProp = ImmutableMap.of( 22 | Sasl.QOP, "auth-conf" 23 | ); 24 | 25 | @Test 26 | public void testSaslDigestMd5() throws Exception { 27 | final SaslClient client = Sasl.createSaslClient( 28 | digestMd5, 29 | "myAuthorizationId", "protocol", "my_realm", qopProp, 30 | new CallbackHandler() { 31 | @Override 32 | public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException { 33 | // Provide realm/user/password, requested user name must be authorized to use 34 | // the username given to createSaslClient static constructor. 35 | for (Callback callback : callbacks) { 36 | if (callback instanceof RealmCallback) { 37 | // realm of the server is defaulted to my_realm 38 | ((RealmCallback) callback).setText("my_realm"); 39 | } else if (callback instanceof NameCallback) { 40 | ((NameCallback) callback).setName("bob"); 41 | } else if (callback instanceof PasswordCallback) { 42 | ((PasswordCallback) callback).setPassword("bob's pass".toCharArray()); 43 | } 44 | } 45 | } 46 | } 47 | ); 48 | final SaslServer server = Sasl.createSaslServer( 49 | digestMd5[0], "protocol", "my_realm", qopProp, 50 | new CallbackHandler() { 51 | @Override 52 | public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException { 53 | String user = null; 54 | PasswordCallback passwordCb = null; 55 | for (Callback callback : callbacks) { 56 | // Set required username and password, framework will verify they match 57 | if (callback instanceof RealmCallback) { 58 | assertThat(((RealmCallback) callback).getDefaultText(), 59 | is(equalTo("my_realm"))); 60 | } else if (callback instanceof NameCallback) { 61 | user = ((NameCallback) callback).getDefaultName(); 62 | } else if (callback instanceof PasswordCallback) { 63 | passwordCb = (PasswordCallback) callback; 64 | } 65 | if (callback instanceof AuthorizeCallback) { 66 | // always allow valid user 67 | final AuthorizeCallback cb = (AuthorizeCallback) callback; 68 | assertThat(cb.getAuthorizationID(), equalTo("myAuthorizationId")); 69 | cb.setAuthorized(true); 70 | } 71 | } 72 | ImmutableMap usersDb = ImmutableMap.of( 73 | "bob", "bob's pass".toCharArray(), 74 | "admin", "secret pass".toCharArray() 75 | ); 76 | if (usersDb.containsKey(user) && passwordCb != null) { 77 | passwordCb.setPassword(usersDb.get(user)); 78 | } 79 | } 80 | } 81 | ); 82 | byte[] resp = new byte[]{}; 83 | assertThat(client.hasInitialResponse(), is(false)); 84 | // in principle there should be a loop here, but we know 85 | // that this MD5-digest implementation is done after 3 rounds 86 | resp = server.evaluateResponse(resp); 87 | resp = client.evaluateChallenge(resp); 88 | resp = server.evaluateResponse(resp); 89 | assertThat(server.isComplete(), is(true)); 90 | assertThat(client.evaluateChallenge(resp), is(nullValue())); 91 | assertThat(client.isComplete(), is(true)); 92 | 93 | byte[] content = "abcdefg".getBytes(Charsets.US_ASCII); 94 | final byte[] wrapped = server.wrap(content, 0, content.length); 95 | assertThat(wrapped, is(not(equalTo(content)))); 96 | assertThat(client.unwrap(wrapped, 0, wrapped.length), 97 | is(equalTo(content))); 98 | } 99 | 100 | } 101 | --------------------------------------------------------------------------------