├── .cproject ├── .gitignore ├── .project ├── .settings ├── .svn │ ├── all-wcprops │ ├── entries │ ├── prop-base │ │ └── org.eclipse.cdt.ui.prefs.svn-base │ └── text-base │ │ ├── org.eclipse.cdt.codan.core.prefs.svn-base │ │ └── org.eclipse.cdt.ui.prefs.svn-base ├── org.eclipse.cdt.codan.core.prefs └── org.eclipse.cdt.ui.prefs ├── Makefile ├── README.md ├── doc ├── CTS_libevent.pdf ├── command.plist ├── constant hash.jpg └── gingko.1 ├── lib ├── libev ├── libev-4.04.tar.gz ├── libev_auxten.patch └── patch_build.sh ├── src ├── AUTHORS ├── ChangeLog ├── Makefile.am ├── NEWS ├── README ├── TODO ├── async_conn.cpp ├── async_pool.h ├── async_threads.cpp ├── autogen.sh ├── clnt_main.cpp ├── clnt_unittest.cpp ├── configure.in ├── erase_job.py ├── gingko.h ├── gingko_base.cpp ├── gingko_clnt.cpp ├── gingko_clnt.h ├── gingko_common.h ├── gingko_serv.cpp ├── gingko_serv.h ├── hash │ ├── AUTHORS │ ├── ChangeLog │ ├── Makefile.am │ ├── NEWS │ ├── README │ ├── configure.in │ ├── gko_zip.cpp │ ├── gko_zip.h │ ├── lz4.cpp │ ├── lz4.h │ ├── md5.cpp │ ├── md5.h │ ├── xor_hash.cpp │ └── xor_hash.h ├── install-sh ├── job_state.h ├── limit.cpp ├── limit.h ├── log.cpp ├── log.h ├── missing ├── option.cpp ├── option.h ├── path.cpp ├── path.h ├── progress.cpp ├── progress.h ├── route.cpp ├── route.h ├── run.sh ├── run2.sh ├── seed.cpp ├── seed.h ├── serv_main.cpp ├── serv_unittest.cpp ├── snap.cpp ├── snap.h ├── socket.cpp ├── socket.h └── unittest.h └── testcase ├── test.sh ├── test.sh.absln └── test.sh.ln /.cproject: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | INSTALL 2 | depcomp 3 | Makefile.in 4 | COPYING 5 | aclocal.m4 6 | bin/ 7 | conf/ 8 | deploy 9 | src/p2p.sh 10 | src/p2pcheck.sh 11 | configure 12 | svn-commit.* 13 | .deps/ 14 | clnt_unittest 15 | serv_unittest 16 | config.h.in 17 | config.h 18 | config.log 19 | config.status 20 | *.tar.gz 21 | src/hash/Makefile 22 | src/Makefile 23 | *.dmg 24 | .DS_Store 25 | autom4te* 26 | gingko_clnt 27 | gingko_serv 28 | stamp-h1 29 | *.orig 30 | kfp_docs* 31 | server.log 32 | client.log 33 | *.[ao] 34 | build/ 35 | *~ 36 | .svn/ 37 | test 38 | output* 39 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | gingko 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.cdt.managedbuilder.core.genmakebuilder 10 | clean,full,incremental, 11 | 12 | 13 | ?name? 14 | 15 | 16 | 17 | org.eclipse.cdt.make.core.append_environment 18 | true 19 | 20 | 21 | org.eclipse.cdt.make.core.autoBuildTarget 22 | all 23 | 24 | 25 | org.eclipse.cdt.make.core.buildArguments 26 | 27 | 28 | 29 | org.eclipse.cdt.make.core.buildCommand 30 | make 31 | 32 | 33 | org.eclipse.cdt.make.core.cleanBuildTarget 34 | clean 35 | 36 | 37 | org.eclipse.cdt.make.core.contents 38 | org.eclipse.cdt.make.core.activeConfigSettings 39 | 40 | 41 | org.eclipse.cdt.make.core.enableAutoBuild 42 | false 43 | 44 | 45 | org.eclipse.cdt.make.core.enableCleanBuild 46 | true 47 | 48 | 49 | org.eclipse.cdt.make.core.enableFullBuild 50 | true 51 | 52 | 53 | org.eclipse.cdt.make.core.fullBuildTarget 54 | all 55 | 56 | 57 | org.eclipse.cdt.make.core.stopOnError 58 | true 59 | 60 | 61 | org.eclipse.cdt.make.core.useDefaultBuildCmd 62 | true 63 | 64 | 65 | 66 | 67 | org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder 68 | full,incremental, 69 | 70 | 71 | 72 | 73 | 74 | org.eclipse.cdt.core.cnature 75 | org.eclipse.cdt.core.ccnature 76 | org.eclipse.cdt.managedbuilder.core.managedBuildNature 77 | org.eclipse.cdt.managedbuilder.core.ScannerConfigNature 78 | 79 | 80 | -------------------------------------------------------------------------------- /.settings/.svn/all-wcprops: -------------------------------------------------------------------------------- 1 | K 25 2 | svn:wc:ra_dav:version-url 3 | V 57 4 | /op/oped/noah/!svn/ver/60093/trunk/tools/gingko/.settings 5 | END 6 | org.eclipse.cdt.codan.core.prefs 7 | K 25 8 | svn:wc:ra_dav:version-url 9 | V 90 10 | /op/oped/noah/!svn/ver/60093/trunk/tools/gingko/.settings/org.eclipse.cdt.codan.core.prefs 11 | END 12 | org.eclipse.cdt.ui.prefs 13 | K 25 14 | svn:wc:ra_dav:version-url 15 | V 82 16 | /op/oped/noah/!svn/ver/45469/trunk/tools/gingko/.settings/org.eclipse.cdt.ui.prefs 17 | END 18 | -------------------------------------------------------------------------------- /.settings/.svn/entries: -------------------------------------------------------------------------------- 1 | 10 2 | 3 | dir 4 | 69182 5 | https://svn.baidu.com/op/oped/noah/trunk/tools/gingko/.settings 6 | https://svn.baidu.com/op/oped/noah 7 | 8 | 9 | 10 | 2011-08-08T08:43:59.747744Z 11 | 60093 12 | wangpengcheng01 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 50f7a8e2-900c-44df-8b5b-4a1fb65e17d8 28 | 29 | org.eclipse.cdt.codan.core.prefs 30 | file 31 | 32 | 33 | 34 | 35 | 2011-08-08T08:47:30.000000Z 36 | 740a5af795b70b0e0452c0719ac2a6af 37 | 2011-08-08T08:43:59.747744Z 38 | 60093 39 | wangpengcheng01 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 6578 62 | 63 | org.eclipse.cdt.ui.prefs 64 | file 65 | 66 | 67 | 68 | 69 | 2011-08-08T08:47:30.000000Z 70 | f446829872c7b49963aebc05b90cbb6c 71 | 2011-06-05T05:25:24.151365Z 72 | 45469 73 | wangpengcheng01 74 | has-props 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 89 96 | 97 | -------------------------------------------------------------------------------- /.settings/.svn/prop-base/org.eclipse.cdt.ui.prefs.svn-base: -------------------------------------------------------------------------------- 1 | K 13 2 | svn:mime-type 3 | V 10 4 | text/plain 5 | END 6 | -------------------------------------------------------------------------------- /.settings/.svn/text-base/org.eclipse.cdt.codan.core.prefs.svn-base: -------------------------------------------------------------------------------- 1 | #Mon Aug 08 10:39:20 CST 2011 2 | eclipse.preferences.version=1 3 | org.eclipse.cdt.codan.checkers.errnoreturn=Warning 4 | org.eclipse.cdt.codan.checkers.errnoreturn.params={implicit\=>false} 5 | org.eclipse.cdt.codan.checkers.errreturnvalue=Error 6 | org.eclipse.cdt.codan.checkers.errreturnvalue.params={} 7 | org.eclipse.cdt.codan.checkers.noreturn=Error 8 | org.eclipse.cdt.codan.checkers.noreturn.params={implicit\=>false} 9 | org.eclipse.cdt.codan.internal.checkers.AbstractClassCreation=Error 10 | org.eclipse.cdt.codan.internal.checkers.AbstractClassCreation.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 11 | org.eclipse.cdt.codan.internal.checkers.AmbiguousProblem=Error 12 | org.eclipse.cdt.codan.internal.checkers.AmbiguousProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 13 | org.eclipse.cdt.codan.internal.checkers.AssignmentInConditionProblem=Warning 14 | org.eclipse.cdt.codan.internal.checkers.AssignmentInConditionProblem.params={} 15 | org.eclipse.cdt.codan.internal.checkers.AssignmentToItselfProblem=Error 16 | org.eclipse.cdt.codan.internal.checkers.AssignmentToItselfProblem.params={} 17 | org.eclipse.cdt.codan.internal.checkers.CaseBreakProblem=Warning 18 | org.eclipse.cdt.codan.internal.checkers.CaseBreakProblem.params={no_break_comment\=>"no break",last_case_param\=>true,empty_case_param\=>false} 19 | org.eclipse.cdt.codan.internal.checkers.CatchByReference=Warning 20 | org.eclipse.cdt.codan.internal.checkers.CatchByReference.params={unknown\=>false,exceptions\=>()} 21 | org.eclipse.cdt.codan.internal.checkers.CircularReferenceProblem=Error 22 | org.eclipse.cdt.codan.internal.checkers.CircularReferenceProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 23 | org.eclipse.cdt.codan.internal.checkers.FieldResolutionProblem=Error 24 | org.eclipse.cdt.codan.internal.checkers.FieldResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 25 | org.eclipse.cdt.codan.internal.checkers.FunctionResolutionProblem=Error 26 | org.eclipse.cdt.codan.internal.checkers.FunctionResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 27 | org.eclipse.cdt.codan.internal.checkers.InvalidArguments=Error 28 | org.eclipse.cdt.codan.internal.checkers.InvalidArguments.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 29 | org.eclipse.cdt.codan.internal.checkers.InvalidTemplateArgumentsProblem=Error 30 | org.eclipse.cdt.codan.internal.checkers.InvalidTemplateArgumentsProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 31 | org.eclipse.cdt.codan.internal.checkers.LabelStatementNotFoundProblem=Error 32 | org.eclipse.cdt.codan.internal.checkers.LabelStatementNotFoundProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 33 | org.eclipse.cdt.codan.internal.checkers.MemberDeclarationNotFoundProblem=Error 34 | org.eclipse.cdt.codan.internal.checkers.MemberDeclarationNotFoundProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 35 | org.eclipse.cdt.codan.internal.checkers.MethodResolutionProblem=Error 36 | org.eclipse.cdt.codan.internal.checkers.MethodResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 37 | org.eclipse.cdt.codan.internal.checkers.NamingConventionFunctionChecker=Info 38 | org.eclipse.cdt.codan.internal.checkers.NamingConventionFunctionChecker.params={pattern\=>"^[a-z]",macro\=>true,exceptions\=>()} 39 | org.eclipse.cdt.codan.internal.checkers.NonVirtualDestructorProblem=Warning 40 | org.eclipse.cdt.codan.internal.checkers.NonVirtualDestructorProblem.params={} 41 | org.eclipse.cdt.codan.internal.checkers.OverloadProblem=Error 42 | org.eclipse.cdt.codan.internal.checkers.OverloadProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 43 | org.eclipse.cdt.codan.internal.checkers.RedeclarationProblem=Error 44 | org.eclipse.cdt.codan.internal.checkers.RedeclarationProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 45 | org.eclipse.cdt.codan.internal.checkers.RedefinitionProblem=Error 46 | org.eclipse.cdt.codan.internal.checkers.RedefinitionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 47 | org.eclipse.cdt.codan.internal.checkers.ReturnStyleProblem=Warning 48 | org.eclipse.cdt.codan.internal.checkers.ReturnStyleProblem.params={} 49 | org.eclipse.cdt.codan.internal.checkers.ScanfFormatStringSecurityProblem=Warning 50 | org.eclipse.cdt.codan.internal.checkers.ScanfFormatStringSecurityProblem.params={} 51 | org.eclipse.cdt.codan.internal.checkers.StatementHasNoEffectProblem=Warning 52 | org.eclipse.cdt.codan.internal.checkers.StatementHasNoEffectProblem.params={macro\=>true,exceptions\=>()} 53 | org.eclipse.cdt.codan.internal.checkers.SuggestedParenthesisProblem=Warning 54 | org.eclipse.cdt.codan.internal.checkers.SuggestedParenthesisProblem.params={paramNot\=>false} 55 | org.eclipse.cdt.codan.internal.checkers.SuspiciousSemicolonProblem=Warning 56 | org.eclipse.cdt.codan.internal.checkers.SuspiciousSemicolonProblem.params={else\=>false,afterelse\=>false} 57 | org.eclipse.cdt.codan.internal.checkers.TypeResolutionProblem=Error 58 | org.eclipse.cdt.codan.internal.checkers.TypeResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 59 | org.eclipse.cdt.codan.internal.checkers.UnusedFunctionDeclarationProblem=Warning 60 | org.eclipse.cdt.codan.internal.checkers.UnusedFunctionDeclarationProblem.params={} 61 | org.eclipse.cdt.codan.internal.checkers.UnusedStaticFunctionProblem=Warning 62 | org.eclipse.cdt.codan.internal.checkers.UnusedStaticFunctionProblem.params={} 63 | org.eclipse.cdt.codan.internal.checkers.UnusedVariableDeclarationProblem=Warning 64 | org.eclipse.cdt.codan.internal.checkers.UnusedVariableDeclarationProblem.params={exceptions\=>("@(\#)","$Id")} 65 | org.eclipse.cdt.codan.internal.checkers.VariableResolutionProblem=Error 66 | org.eclipse.cdt.codan.internal.checkers.VariableResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 67 | -------------------------------------------------------------------------------- /.settings/.svn/text-base/org.eclipse.cdt.ui.prefs.svn-base: -------------------------------------------------------------------------------- 1 | #Sun Jun 05 13:16:25 CST 2011 2 | eclipse.preferences.version=1 3 | formatter_settings_version=1 4 | -------------------------------------------------------------------------------- /.settings/org.eclipse.cdt.codan.core.prefs: -------------------------------------------------------------------------------- 1 | #Mon Aug 08 10:39:20 CST 2011 2 | eclipse.preferences.version=1 3 | org.eclipse.cdt.codan.checkers.errnoreturn=Warning 4 | org.eclipse.cdt.codan.checkers.errnoreturn.params={implicit\=>false} 5 | org.eclipse.cdt.codan.checkers.errreturnvalue=Error 6 | org.eclipse.cdt.codan.checkers.errreturnvalue.params={} 7 | org.eclipse.cdt.codan.checkers.noreturn=Error 8 | org.eclipse.cdt.codan.checkers.noreturn.params={implicit\=>false} 9 | org.eclipse.cdt.codan.internal.checkers.AbstractClassCreation=Error 10 | org.eclipse.cdt.codan.internal.checkers.AbstractClassCreation.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 11 | org.eclipse.cdt.codan.internal.checkers.AmbiguousProblem=Error 12 | org.eclipse.cdt.codan.internal.checkers.AmbiguousProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 13 | org.eclipse.cdt.codan.internal.checkers.AssignmentInConditionProblem=Warning 14 | org.eclipse.cdt.codan.internal.checkers.AssignmentInConditionProblem.params={} 15 | org.eclipse.cdt.codan.internal.checkers.AssignmentToItselfProblem=Error 16 | org.eclipse.cdt.codan.internal.checkers.AssignmentToItselfProblem.params={} 17 | org.eclipse.cdt.codan.internal.checkers.CaseBreakProblem=Warning 18 | org.eclipse.cdt.codan.internal.checkers.CaseBreakProblem.params={no_break_comment\=>"no break",last_case_param\=>true,empty_case_param\=>false} 19 | org.eclipse.cdt.codan.internal.checkers.CatchByReference=Warning 20 | org.eclipse.cdt.codan.internal.checkers.CatchByReference.params={unknown\=>false,exceptions\=>()} 21 | org.eclipse.cdt.codan.internal.checkers.CircularReferenceProblem=Error 22 | org.eclipse.cdt.codan.internal.checkers.CircularReferenceProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 23 | org.eclipse.cdt.codan.internal.checkers.FieldResolutionProblem=Error 24 | org.eclipse.cdt.codan.internal.checkers.FieldResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 25 | org.eclipse.cdt.codan.internal.checkers.FunctionResolutionProblem=Error 26 | org.eclipse.cdt.codan.internal.checkers.FunctionResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 27 | org.eclipse.cdt.codan.internal.checkers.InvalidArguments=Error 28 | org.eclipse.cdt.codan.internal.checkers.InvalidArguments.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 29 | org.eclipse.cdt.codan.internal.checkers.InvalidTemplateArgumentsProblem=Error 30 | org.eclipse.cdt.codan.internal.checkers.InvalidTemplateArgumentsProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 31 | org.eclipse.cdt.codan.internal.checkers.LabelStatementNotFoundProblem=Error 32 | org.eclipse.cdt.codan.internal.checkers.LabelStatementNotFoundProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 33 | org.eclipse.cdt.codan.internal.checkers.MemberDeclarationNotFoundProblem=Error 34 | org.eclipse.cdt.codan.internal.checkers.MemberDeclarationNotFoundProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 35 | org.eclipse.cdt.codan.internal.checkers.MethodResolutionProblem=Error 36 | org.eclipse.cdt.codan.internal.checkers.MethodResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 37 | org.eclipse.cdt.codan.internal.checkers.NamingConventionFunctionChecker=Info 38 | org.eclipse.cdt.codan.internal.checkers.NamingConventionFunctionChecker.params={pattern\=>"^[a-z]",macro\=>true,exceptions\=>()} 39 | org.eclipse.cdt.codan.internal.checkers.NonVirtualDestructorProblem=Warning 40 | org.eclipse.cdt.codan.internal.checkers.NonVirtualDestructorProblem.params={} 41 | org.eclipse.cdt.codan.internal.checkers.OverloadProblem=Error 42 | org.eclipse.cdt.codan.internal.checkers.OverloadProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 43 | org.eclipse.cdt.codan.internal.checkers.RedeclarationProblem=Error 44 | org.eclipse.cdt.codan.internal.checkers.RedeclarationProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 45 | org.eclipse.cdt.codan.internal.checkers.RedefinitionProblem=Error 46 | org.eclipse.cdt.codan.internal.checkers.RedefinitionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 47 | org.eclipse.cdt.codan.internal.checkers.ReturnStyleProblem=Warning 48 | org.eclipse.cdt.codan.internal.checkers.ReturnStyleProblem.params={} 49 | org.eclipse.cdt.codan.internal.checkers.ScanfFormatStringSecurityProblem=Warning 50 | org.eclipse.cdt.codan.internal.checkers.ScanfFormatStringSecurityProblem.params={} 51 | org.eclipse.cdt.codan.internal.checkers.StatementHasNoEffectProblem=Warning 52 | org.eclipse.cdt.codan.internal.checkers.StatementHasNoEffectProblem.params={macro\=>true,exceptions\=>()} 53 | org.eclipse.cdt.codan.internal.checkers.SuggestedParenthesisProblem=Warning 54 | org.eclipse.cdt.codan.internal.checkers.SuggestedParenthesisProblem.params={paramNot\=>false} 55 | org.eclipse.cdt.codan.internal.checkers.SuspiciousSemicolonProblem=Warning 56 | org.eclipse.cdt.codan.internal.checkers.SuspiciousSemicolonProblem.params={else\=>false,afterelse\=>false} 57 | org.eclipse.cdt.codan.internal.checkers.TypeResolutionProblem=Error 58 | org.eclipse.cdt.codan.internal.checkers.TypeResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 59 | org.eclipse.cdt.codan.internal.checkers.UnusedFunctionDeclarationProblem=Warning 60 | org.eclipse.cdt.codan.internal.checkers.UnusedFunctionDeclarationProblem.params={} 61 | org.eclipse.cdt.codan.internal.checkers.UnusedStaticFunctionProblem=Warning 62 | org.eclipse.cdt.codan.internal.checkers.UnusedStaticFunctionProblem.params={} 63 | org.eclipse.cdt.codan.internal.checkers.UnusedVariableDeclarationProblem=Warning 64 | org.eclipse.cdt.codan.internal.checkers.UnusedVariableDeclarationProblem.params={exceptions\=>("@(\#)","$Id")} 65 | org.eclipse.cdt.codan.internal.checkers.VariableResolutionProblem=Error 66 | org.eclipse.cdt.codan.internal.checkers.VariableResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>false,RUN_ON_INC_BUILD\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 67 | -------------------------------------------------------------------------------- /.settings/org.eclipse.cdt.ui.prefs: -------------------------------------------------------------------------------- 1 | #Sun Jun 05 13:16:25 CST 2011 2 | eclipse.preferences.version=1 3 | formatter_settings_version=1 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all makelib compile rm_config clean move check 2 | all: check_configure makelib compile #move rm_config 3 | 4 | check_configure: 5 | if [ ! -x ./src/configure ]; then ( cd ./src && ./autogen.sh ); fi 6 | 7 | makelib: 8 | cd lib && bash -x patch_build.sh && cd .. 9 | 10 | compile: 11 | #cd src && ./configure CXXFLAGS='-ggdb -D_GKO_VERSION=\"$(subst VERSION:,,$(VERSION_SCMPF))\"' --enable-debug -enable-unittest && make clean && 12 | cd src && ./configure CXXFLAGS='-ggdb ' --enable-debug && make clean && \ 13 | cd hash && make && cd .. && make -j 4 && cd .. 14 | 15 | rm_config: 16 | rm ./src/config.h 17 | rm ./src/hash/config.h 18 | rm -rf ./lib/libev/include 19 | 20 | clean: 21 | pwd 22 | rm -rf output test 23 | cd src && if [ -f Makefile ];then make clean; fi 24 | #find ./../../../../../../.. -type f -name "event.h" 25 | 26 | move: 27 | if [ ! -d output ];then mkdir output;fi 28 | cd output && if [ ! -d bin ];then mkdir bin; fi && \ 29 | if [ ! -d testbin ];then mkdir testbin; fi && \ 30 | if [ ! -d conf ];then mkdir conf; fi 31 | cp ./src/gingko_serv ./output/bin/gkod 32 | cp ./src/gingko_clnt ./output/bin/gkocp 33 | cp ./src/serv_unittest ./output/testbin/ 34 | cp ./src/clnt_unittest ./output/testbin/ 35 | cp -r ./output/testbin ./test 36 | cp ./bin/* ./output/bin/ 37 | cp ./conf/* ./output/conf/ 38 | #cp ./src/erase_job.py ./output/bin/ 39 | #cp ./src/run2.sh ./output/bin/gkod_ctl 40 | cp deploy ./output/ && chmod +x ./output/deploy 41 | cd output && md5sum deploy bin/* conf/* > md5sum 42 | #cd output/bin && cp gkocp{,.new} 43 | #cd output/bin && cp gkod{,.new} 44 | #cd output/bin && cp gkod_ctl{,.new} 45 | chmod +x ./output/bin/* 46 | cd output && tar czvf gingko.tgz bin conf md5sum deploy 47 | 48 | check: 49 | cd test && ./clnt_unittest && ./serv_unittest 50 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | src/README -------------------------------------------------------------------------------- /doc/CTS_libevent.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auxten/gingko/ceeeaaa9bb4ee7e6cde43f45f80f4e7c71f97ff2/doc/CTS_libevent.pdf -------------------------------------------------------------------------------- /doc/command.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | GET 6 | 7 | Description 8 | client请求blocks 9 | Command 10 | GET block_id 11 | Function 12 | int get_blocks(long block_id, int job_id) 13 | 14 | QUIT 15 | 16 | Description 17 | client退出job 18 | Command 19 | QUIT job_id 20 | Function 21 | int quit_job(int job_id) 22 | 23 | JOIN 24 | 25 | Description 26 | client加入job 27 | Command 28 | JOIN uri 29 | Function 30 | int join_job(char * uri) 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /doc/constant hash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auxten/gingko/ceeeaaa9bb4ee7e6cde43f45f80f4e7c71f97ff2/doc/constant hash.jpg -------------------------------------------------------------------------------- /doc/gingko.1: -------------------------------------------------------------------------------- 1 | .\"Modified from man(1) of FreeBSD, the NetBSD mdoc.template, and mdoc.samples. 2 | .\"See Also: 3 | .\"man mdoc.samples for a complete listing of options 4 | .\"man mdoc for the short list of editing options 5 | .\"/usr/share/misc/mdoc.template 6 | .Dd 11-3-31 \" DATE 7 | .Dt gingko 1 \" Program name and manual section number 8 | .Os Darwin 9 | .Sh NAME \" Section Header - required - don't modify 10 | .Nm gingko, 11 | .\" The following lines are read in generating the apropos(man -k) database. Use only key 12 | .\" words here as the database is built based on the words here and in the .ND line. 13 | .Nm Other_name_for_same_program(), 14 | .Nm Yet another name for the same program. 15 | .\" Use .Nm macro to designate other names for the documented program. 16 | .Nd This line parsed for whatis database. 17 | .Sh SYNOPSIS \" Section Header - required - don't modify 18 | .Nm 19 | .Op Fl abcd \" [-abcd] 20 | .Op Fl a Ar path \" [-a path] 21 | .Op Ar file \" [file] 22 | .Op Ar \" [file ...] 23 | .Ar arg0 \" Underlined argument - use .Ar anywhere to underline 24 | arg2 ... \" Arguments 25 | .Sh DESCRIPTION \" Section Header - required - don't modify 26 | Use the .Nm macro to refer to your program throughout the man page like such: 27 | .Nm 28 | Underlining is accomplished with the .Ar macro like this: 29 | .Ar underlined text . 30 | .Pp \" Inserts a space 31 | A list of items with descriptions: 32 | .Bl -tag -width -indent \" Begins a tagged list 33 | .It item a \" Each item preceded by .It macro 34 | Description of item a 35 | .It item b 36 | Description of item b 37 | .El \" Ends the list 38 | .Pp 39 | A list of flags and their descriptions: 40 | .Bl -tag -width -indent \" Differs from above in tag removed 41 | .It Fl a \"-a flag as a list item 42 | Description of -a flag 43 | .It Fl b 44 | Description of -b flag 45 | .El \" Ends the list 46 | .Pp 47 | .\" .Sh ENVIRONMENT \" May not be needed 48 | .\" .Bl -tag -width "ENV_VAR_1" -indent \" ENV_VAR_1 is width of the string ENV_VAR_1 49 | .\" .It Ev ENV_VAR_1 50 | .\" Description of ENV_VAR_1 51 | .\" .It Ev ENV_VAR_2 52 | .\" Description of ENV_VAR_2 53 | .\" .El 54 | .Sh FILES \" File used or created by the topic of the man page 55 | .Bl -tag -width "/Users/joeuser/Library/really_long_file_name" -compact 56 | .It Pa /usr/share/file_name 57 | FILE_1 description 58 | .It Pa /Users/joeuser/Library/really_long_file_name 59 | FILE_2 description 60 | .El \" Ends the list 61 | .\" .Sh DIAGNOSTICS \" May not be needed 62 | .\" .Bl -diag 63 | .\" .It Diagnostic Tag 64 | .\" Diagnostic informtion here. 65 | .\" .It Diagnostic Tag 66 | .\" Diagnostic informtion here. 67 | .\" .El 68 | .Sh SEE ALSO 69 | .\" List links in ascending order by section, alphabetically within a section. 70 | .\" Please do not reference files that do not exist without filing a bug report 71 | .Xr a 1 , 72 | .Xr b 1 , 73 | .Xr c 1 , 74 | .Xr a 2 , 75 | .Xr b 2 , 76 | .Xr a 3 , 77 | .Xr b 3 78 | .\" .Sh BUGS \" Document known, unremedied bugs 79 | .\" .Sh HISTORY \" Document history if command behaves in a unique manner -------------------------------------------------------------------------------- /lib/libev: -------------------------------------------------------------------------------- 1 | libev-4.04fixed -------------------------------------------------------------------------------- /lib/libev-4.04.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auxten/gingko/ceeeaaa9bb4ee7e6cde43f45f80f4e7c71f97ff2/lib/libev-4.04.tar.gz -------------------------------------------------------------------------------- /lib/libev_auxten.patch: -------------------------------------------------------------------------------- 1 | diff -upNr libev-4.04/ev.c libev-4.04-fixed/ev.c 2 | --- libev-4.04/ev.c 2011-02-09 07:17:37.000000000 +0800 3 | +++ libev-4.04-fixed/ev.c 2011-09-07 15:29:55.000000000 +0800 4 | @@ -948,6 +948,8 @@ fd_event_nocheck (EV_P_ int fd, int reve 5 | 6 | if (ev) 7 | ev_feed_event (EV_A_ (W)w, ev); 8 | + if (w == (ev_io *)((WL)w)->next) 9 | + break; 10 | } 11 | } 12 | 13 | @@ -1017,7 +1019,11 @@ fd_reify (EV_P) 14 | anfd->events = 0; 15 | 16 | for (w = (ev_io *)anfd->head; w; w = (ev_io *)((WL)w)->next) 17 | - anfd->events |= (unsigned char)w->events; 18 | + { 19 | + anfd->events |= (unsigned char)w->events; 20 | + if (w == (ev_io *)((WL)w)->next) 21 | + break; 22 | + } 23 | 24 | if (o_events != anfd->events) 25 | o_reify = EV__IOFDSET; /* actually |= */ 26 | @@ -1495,6 +1501,8 @@ child_reap (EV_P_ int chain, int pid, in 27 | w->rstatus = status; 28 | ev_feed_event (EV_A_ (W)w, EV_CHILD); 29 | } 30 | + if (w == (ev_child *)((WL)w)->next) 31 | + break; 32 | } 33 | } 34 | 35 | -------------------------------------------------------------------------------- /lib/patch_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | #/bin/sh 4 | 5 | #specify where to install by PREFIX or COMMAND-LINE ARG1 6 | PREFIX="$PWD/libev-4.04fixed" 7 | if [ -n "$1" ]; then 8 | PREFIX=$1 9 | fi 10 | SRCDIR=libev-4.04 11 | LIBNAME="libev-4.04" 12 | if [ -d "$SRCDIR" ]; then 13 | rm -rf $SRCDIR 14 | fi 15 | file=$LIBNAME.tar.gz 16 | if [ ! -f "$file" ]; then 17 | echo "error: no $file" 18 | exit 1 19 | fi 20 | tar xvzf $file 21 | if [ ! -d "$SRCDIR" ]; then 22 | echo "error: no $SRCDIR" 23 | exit 1 24 | fi 25 | 26 | cp libev_auxten.patch $SRCDIR/libev_auxten.patch 27 | cd $SRCDIR 28 | patch -p1 < libev_auxten.patch 29 | 30 | ./configure --prefix=$PREFIX --enable-shared=no --enable-static 31 | 32 | make; make install 33 | cd .. 34 | 35 | rm -rf libev 36 | ln -s libev-4.04fixed libev 37 | rm -rf $SRCDIR 38 | 39 | echo "done!" 40 | 41 | 42 | -------------------------------------------------------------------------------- /src/AUTHORS: -------------------------------------------------------------------------------- 1 | auxtenwpc@gmail.com 2 | -------------------------------------------------------------------------------- /src/ChangeLog: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auxten/gingko/ceeeaaa9bb4ee7e6cde43f45f80f4e7c71f97ff2/src/ChangeLog -------------------------------------------------------------------------------- /src/Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = hash . 2 | INCLUDES = -I$(srcdir)/src\ 3 | -I$(srcdir)/hash \ 4 | -include config.h \ 5 | -I$(srcdir)/../lib/libev/include 6 | # -I$(srcdir)/../../../../../../third-64/libev/include 7 | # -I$(srcdir)/../../../../../../../third-64/libev/include 8 | #export INCLUDES 9 | 10 | noinst_PROGRAMS=gingko_serv gingko_clnt 11 | 12 | clnt_unittest_SOURCES=async_threads.cpp async_conn.cpp \ 13 | gingko_base.cpp path.cpp log.cpp option.cpp socket.cpp limit.cpp \ 14 | route.cpp snap.cpp clnt_unittest.cpp gingko_clnt.cpp progress.cpp 15 | clnt_unittest_LDADD=$(srcdir)/../lib/libev/lib/libev.a \ 16 | $(srcdir)/../../../../../../third-64/gtest/lib/libgtest.a \ 17 | $(srcdir)/hash/libxor_hash.a \ 18 | $(srcdir)/hash/libgko_zip.a 19 | clnt_unittest_LDFLAGS=-D_GNU_SOURCE -ggdb -funsigned-char 20 | 21 | serv_unittest_SOURCES=async_threads.cpp async_conn.cpp \ 22 | gingko_base.cpp path.cpp log.cpp option.cpp socket.cpp limit.cpp \ 23 | seed.cpp hash/xor_hash.cpp serv_unittest.cpp gingko_serv.cpp 24 | serv_unittest_LDADD=$(srcdir)/../lib/libev/lib/libev.a \ 25 | $(srcdir)/../../../../../../third-64/gtest/lib/libgtest.a \ 26 | $(srcdir)/hash/libxor_hash.a \ 27 | $(srcdir)/hash/libgko_zip.a 28 | serv_unittest_LDFLAGS=-D_GNU_SOURCE -ggdb -funsigned-char 29 | 30 | gingko_serv_SOURCES=async_threads.cpp async_conn.cpp gingko_serv.cpp serv_main.cpp\ 31 | gingko_base.cpp path.cpp log.cpp option.cpp socket.cpp limit.cpp \ 32 | seed.cpp 33 | gingko_serv_LDADD=$(srcdir)/../lib/libev/lib/libev.a \ 34 | $(srcdir)/hash/libxor_hash.a \ 35 | $(srcdir)/hash/libgko_zip.a 36 | gingko_serv_LDFLAGS=-D_GNU_SOURCE -ggdb -funsigned-char -lpthread 37 | 38 | gingko_clnt_SOURCES=async_threads.cpp async_conn.cpp gingko_clnt.cpp clnt_main.cpp\ 39 | gingko_base.cpp path.cpp log.cpp option.cpp socket.cpp limit.cpp \ 40 | route.cpp snap.cpp progress.cpp 41 | gingko_clnt_LDADD=$(srcdir)/../lib/libev/lib/libev.a \ 42 | $(srcdir)/hash/libxor_hash.a \ 43 | $(srcdir)/hash/libgko_zip.a 44 | gingko_clnt_LDFLAGS=-D_GNU_SOURCE -ggdb -funsigned-char -lpthread 45 | 46 | DEFS=-D_GNU_SOURCE -ggdb -funsigned-char 47 | -------------------------------------------------------------------------------- /src/NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auxten/gingko/ceeeaaa9bb4ee7e6cde43f45f80f4e7c71f97ff2/src/NEWS -------------------------------------------------------------------------------- /src/README: -------------------------------------------------------------------------------- 1 | SEE: http://auxten.github.io/gingko/ 2 | -------------------------------------------------------------------------------- /src/TODO: -------------------------------------------------------------------------------- 1 | 线程池可以扩大 2 | 线程缓冲区做成缓冲池 3 | # client的ip以及port getpeername inet_ntoa 4 | # zero file core 5 | # sendfile BSD linux不太相同可以考虑采用mmap ＋ write See: http://blog.sina.com.cn/s/blog_4967ea190100ja5p.html 6 | # find port to listen 7 | # strerror strerror_r必要？ 8 | # 跨文件的数据块校验失败 9 | # libevent读文件event_init event_base_del效率低也许select更好增大块大小可以缓解 10 | # 多线程xor_hash 11 | # shutdown socket 12 | # socket 超时 13 | # 最后请求server 14 | # vnode 使用线程 15 | # sendblock broken pipe 死锁 16 | # 限速 17 | # not ready_to_serv host set保存host 18 | # 获得可用源就不再去请求server 19 | # client 线程数限制，find no client slot what response？ 20 | # 心跳监测 21 | # 路径处理，目的文件夹不存在 22 | # server到不了就会hang 23 | # help函数 24 | # SO_RCVBUF 25 | # option pool大小 26 | # quit_job_s 如果host不在的逻辑 27 | # serv pool 满了的心跳检测 28 | # 退出任务广播 29 | # 单文件支持 30 | # join 超时探测 31 | # join 进度传递 32 | # getopt_long() 33 | # 续传 34 | # 在sheldon上 recv 的 MSG_WAITALL 貌似不起作用 35 | # xor_hash过程中 job erase 导致core https://computing.llnl.gov/tutorials/pthreads/man/pthread_setcanceltype.txt 36 | # 迁移到libev？ 37 | # 异常文件处理 38 | # job—state 39 | # client日志默认输出至stderr 40 | # client正常退出打印一条日志 41 | # client做种时间可以配置 42 | # 续传模式在目的文件被mv走可能引发的bug 43 | # sig handler 44 | # dead不彻底摘除增加时延 pthread_cancel 用时◎◎◎ 45 | 46 | free all job 函数 47 | 消除满地的exit 48 | 在二期增加--follow-link开关，一期只采用不follow的处理方式 49 | 做种限速 50 | 51 | 52 | 路径通配符 53 | Server单点消除 54 | 源路径是软链接 55 | 流式P2P 56 | job 过期 57 | # xor_hash int 读提速没有作用 58 | # sep_arg 返回值检查 59 | # JOIN 不存在的目录引发 clnt core 60 | 61 | ###############profile############### 62 | Flat profile: 63 | 64 | Each sample counts as 0.01 seconds. 65 | % cumulative self self total 66 | time seconds seconds calls ms/call ms/call name 67 | 98.27 0.85 0.85 213 3.97 3.97 digest_ok(void*, _s_block*) 68 | 1.16 0.86 0.01 vnode_download(void*) 69 | 0.58 0.86 0.01 175 0.03 0.03 writeblock(_s_job*, unsigned char const*, _s_block*) 70 | 0.00 0.86 0.00 159 0.00 0.00 bw_down_limit(int) 71 | 0.00 0.86 0.00 62 0.00 0.00 ev_fn_write(int, short, void*) 72 | 0.00 0.86 0.00 62 0.00 0.00 gethostname_my(char const*, hostent*, char**, unsigned long) 73 | 0.00 0.86 0.00 62 0.00 0.00 sendall(int, void const*, int, int) 74 | 0.00 0.86 0.00 61 0.00 0.00 connect_host(_s_host*, int, int) 75 | 0.00 0.86 0.00 61 0.00 0.00 sep_arg(char*, char**, int) 76 | 0.00 0.86 0.00 59 0.00 13.12 get_blocks_c(_s_host*, long, long, unsigned char, char*) 77 | 0.00 0.86 0.00 54 0.00 0.00 close_socket(int) 78 | 0.00 0.86 0.00 54 0.00 0.00 cmpByDistance(_s_host const&, _s_host const&) 79 | 0.00 0.86 0.00 45 0.00 0.00 operator==(_s_host const&, _s_host const&) 80 | 0.00 0.86 0.00 36 0.00 0.00 __gnu_cxx::__normal_iterator<_s_host*, std::vector<_s_host, std::allocator<_s_host> > > std::__find<__gnu_cxx::__normal_iterator<_s_host*, std::vector<_s_host, std::allocator<_s_host> > >, _s_host>(__gnu_cxx::__normal_iterator<_s_host*, std::vector<_s_host, std::allocator<_s_host> > >, __gnu_cxx::__normal_iterator<_s_host*, std::vector<_s_host, std::allocator<_s_host> > >, _s_host const&, std::random_access_iterator_tag) 81 | 0.00 0.86 0.00 27 0.00 19.34 decide_src(_s_job*, int, long, std::vector<_s_host, std::allocator<_s_host> >*, _s_host*) 82 | 0.00 0.86 0.00 27 0.00 0.00 get_blk_src(_s_job*, int, long, std::vector<_s_host, std::allocator<_s_host> >*) 83 | 0.00 0.86 0.00 20 0.00 0.00 thread_worker_new(int) 84 | 0.00 0.86 0.00 19 0.00 0.00 gettimestr(char*) 85 | 0.00 0.86 0.00 19 0.00 0.00 gko_log(WARNING, char const*, ...) 86 | 0.00 0.86 0.00 18 0.00 0.00 std::_Rb_tree<_s_host, _s_host, std::_Identity<_s_host>, std::less<_s_host>, std::allocator<_s_host> >::_M_insert_unique(_s_host const&) 87 | 0.00 0.86 0.00 12 0.00 0.00 conn_client_clear(conn_client*) 88 | 0.00 0.86 0.00 9 0.00 0.00 conn_client_free(conn_client*) 89 | 0.00 0.86 0.00 9 0.00 0.00 conn_tcp_server_on_data(int, short, void*) 90 | 0.00 0.86 0.00 9 0.00 0.00 parse_req(char*) 91 | 0.00 0.86 0.00 9 0.00 0.00 std::vector<_s_host, std::allocator<_s_host> >::_M_insert_aux(__gnu_cxx::__normal_iterator<_s_host*, std::vector<_s_host, std::allocator<_s_host> > >, _s_host const&) 92 | 0.00 0.86 0.00 8 0.00 0.00 conn_send_data(int, void*, unsigned int) 93 | 0.00 0.86 0.00 8 0.00 0.00 std::_Rb_tree<_s_host, _s_host, std::_Identity<_s_host>, std::less<_s_host>, std::allocator<_s_host> >::_M_insert_(std::_Rb_tree_node_base const*, std::_Rb_tree_node_base const*, _s_host const&) 94 | 0.00 0.86 0.00 7 0.00 0.00 conn_setnonblock(int) 95 | 0.00 0.86 0.00 6 0.00 0.00 sendblocks(int, _s_job*, long, long) 96 | 0.00 0.86 0.00 6 0.00 0.00 get_blocks_s(void*, int) 97 | 0.00 0.86 0.00 6 0.00 0.00 add_new_conn_client(int) 98 | 0.00 0.86 0.00 6 0.00 0.00 conn_client_list_get(int) 99 | 0.00 0.86 0.00 6 0.00 0.00 thread_list_find_next() 100 | 0.00 0.86 0.00 6 0.00 0.00 thread_worker_process(int, short, void*) 101 | 0.00 0.86 0.00 6 0.00 0.00 conn_tcp_server_accept(int, short, void*) 102 | 0.00 0.86 0.00 6 0.00 0.00 thread_worker_dispatch(int) 103 | 0.00 0.86 0.00 6 0.00 0.00 host_hash(_s_job*, _s_host const*, _s_host_hash_result*, char) 104 | 0.00 0.86 0.00 5 0.00 0.00 inplace_strip_tailing_slash(char*) 105 | 0.00 0.86 0.00 4 0.00 0.00 std::_Rb_tree<_s_host, _s_host, std::_Identity<_s_host>, std::less<_s_host>, std::allocator<_s_host> >::erase(_s_host const&) 106 | 0.00 0.86 0.00 3 0.00 0.00 std::_Rb_tree<_s_host, _s_host, std::_Identity<_s_host>, std::less<_s_host>, std::allocator<_s_host> >::_M_erase(std::_Rb_tree_node<_s_host>*) 107 | 0.00 0.86 0.00 2 0.00 0.00 merge_path(char*, char const*, char const*) 108 | 0.00 0.86 0.00 2 0.00 0.00 new_host_s(void*, int) 109 | 0.00 0.86 0.00 2 0.00 0.00 inplace_change_path(char*, char const*, char const*) 110 | 0.00 0.86 0.00 2 0.00 0.00 base_name(char*, char const*) 111 | 112 | Flat profile: 113 | Each sample counts as 0.01 seconds. 114 | % cumulative self self total 115 | time seconds seconds calls ms/call ms/call name 116 | 100.01 2.05 2.05 596 3.44 3.44 xor_hash_file(unsigned int, _IO_FILE*, long*, unsigned long*, unsigned char*) 117 | 0.00 2.05 0.00 3054 0.00 0.00 conn_client_free(conn_client*) 118 | 0.00 2.05 0.00 1474 0.00 0.00 ev_fn_gsendfile(int, short, void*) 119 | 0.00 2.05 0.00 248 0.00 0.00 sendfileall(int, int, long*, unsigned long*) 120 | 0.00 2.05 0.00 107 0.00 0.00 conn_client_clear(conn_client*) 121 | 0.00 2.05 0.00 62 0.00 0.00 sendall(int, void const*, int, int) 122 | 0.00 2.05 0.00 61 0.00 0.00 ev_fn_write(int, short, void*) 123 | 0.00 2.05 0.00 55 0.00 0.00 conn_setnonblock(int) 124 | 0.00 2.05 0.00 54 0.00 0.00 conn_send_data(int, void*, unsigned int) 125 | 0.00 2.05 0.00 54 0.00 0.00 add_new_conn_client(int) 126 | 0.00 2.05 0.00 54 0.00 0.00 conn_client_list_get(int) 127 | 0.00 2.05 0.00 54 0.00 0.00 thread_list_find_next() 128 | 0.00 2.05 0.00 54 0.00 0.00 thread_worker_process(int, short, void*) 129 | 0.00 2.05 0.00 54 0.00 0.00 thread_worker_dispatch(int) 130 | 0.00 2.05 0.00 54 0.00 0.00 conn_tcp_server_on_data(int, short, void*) 131 | 0.00 2.05 0.00 53 0.00 0.00 conn_tcp_server_accept(int, short, void*) 132 | 0.00 2.05 0.00 53 0.00 0.00 parse_req(char*) 133 | 0.00 2.05 0.00 52 0.00 0.00 sep_arg(char*, char**, int) 134 | 0.00 2.05 0.00 48 0.00 0.00 sendblocks(int, _s_job*, long, long) 135 | 0.00 2.05 0.00 48 0.00 0.00 get_blocks_s(void*, int) 136 | 0.00 2.05 0.00 20 0.00 0.00 thread_worker_new(int) 137 | 0.00 2.05 0.00 5 0.00 0.00 connect_host(_s_host*, int, int) 138 | 0.00 2.05 0.00 5 0.00 0.00 gethostname_my(char const*, hostent*, char**, unsigned long) 139 | 0.00 2.05 0.00 5 0.00 0.00 sendcmd(_s_host*, char const*, int, int) 140 | 0.00 2.05 0.00 4 0.00 0.00 close_socket(int) 141 | 0.00 2.05 0.00 2 0.00 0.00 join_job_s(void*, int) 142 | 0.00 2.05 0.00 2 0.00 0.00 quit_job_s(void*, int) 143 | 0.00 2.05 0.00 2 0.00 0.00 helo_serv_s(void*, int) 144 | 0.00 2.05 0.00 2 0.00 0.00 broadcast_join(_s_host*, _s_host*) 145 | 0.00 2.05 0.00 2 0.00 0.00 std::_Rb_tree<_s_host, _s_host, std::_Identity<_s_host>, std::less<_s_host>, std::allocator<_s_host> >::_M_insert_(std::_Rb_tree_node_base const*, std::_Rb_tree_node_base const*, _s_host const&) 146 | 0.00 2.05 0.00 2 0.00 0.00 std::_Rb_tree<_s_host, _s_host, std::_Identity<_s_host>, std::less<_s_host>, std::allocator<_s_host> >::erase(_s_host const&) 147 | 0.00 2.05 0.00 2 0.00 0.00 std::_Rb_tree<_s_host, _s_host, std::_Identity<_s_host>, std::less<_s_host>, std::allocator<_s_host> >::_M_erase(std::_Rb_tree_node<_s_host>*) 148 | 0.00 2.05 0.00 2 0.00 0.00 std::_Rb_tree, std::_Select1st >, std::less, std::allocator > >::_M_erase(std::_Rb_tree_node >*) 149 | 0.00 2.05 0.00 1 0.00 0.00 global constructors keyed to grand_lock 150 | 0.00 2.05 0.00 1 0.00 0.00 conn_close() 151 | 0.00 2.05 0.00 1 0.00 0.00 recurse_dir(_s_job*) 152 | 0.00 2.05 0.00 1 0.00 0.00 thread_init() 153 | 0.00 2.05 0.00 1 0.00 0.00 xor_hash_all(_s_job*, long) 154 | 0.00 2.05 0.00 1 0.00 0.00 get_file_count(char const*) 155 | 0.00 2.05 0.00 1 0.00 0.00 conn_tcp_server(conn_server*) 156 | 0.00 2.05 0.00 1 0.00 0.00 conn_client_list_init() 157 | 0.00 2.05 0.00 1 0.00 0.00 set_sig() 158 | 0.00 2.05 0.00 1 0.00 0.00 base_init() 159 | 0.00 2.05 0.00 1 0.00 0.00 erase_job(std::string&) 160 | 0.00 2.05 0.00 1 0.00 0.00 std::_Rb_tree, std::_Select1st >, std::less, std::allocator > >::_M_insert_(std::_Rb_tree_node_base const*, std::_Rb_tree_node_base const*, std::pair const&) 161 | 0.00 2.05 0.00 1 0.00 0.00 std::_Rb_tree, std::_Select1st >, std::less, std::allocator > >::_M_insert_unique(std::pair const&) 162 | 0.00 2.05 0.00 1 0.00 0.00 std::_Rb_tree, std::_Select1st >, std::less, std::allocator 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #include "event.h" 29 | 30 | /// Thread worker 31 | struct thread_worker 32 | { 33 | int id; 34 | pthread_t tid; 35 | struct event_base *ev_base; 36 | struct event ev_notify; 37 | int notify_recv_fd; 38 | int notify_send_fd; 39 | }; 40 | 41 | /// Connection client 42 | struct conn_client 43 | { 44 | int id; 45 | int client_fd; 46 | unsigned long client_addr; 47 | int client_port; 48 | unsigned int conn_time; 49 | func_t handle_client; 50 | struct event ev_read; 51 | char *read_buffer; 52 | unsigned int buffer_size; 53 | }; 54 | 55 | /// Connection server struct 56 | struct conn_server 57 | { 58 | char is_server; 59 | int listen_fd; 60 | struct sockaddr_in listen_addr; 61 | in_addr_t srv_addr; 62 | int srv_port; 63 | unsigned int start_time; 64 | int nonblock; 65 | int listen_queue_length; 66 | int tcp_send_buffer_size; 67 | int tcp_recv_buffer_size; 68 | int send_timeout; 69 | int tcp_reuse; 70 | int tcp_nodelay; 71 | struct event ev_accept; 72 | void (* on_data_callback)(int, void *, unsigned int); 73 | }; 74 | 75 | class gko_pool 76 | { 77 | public: 78 | static s_host_t gko_serv; 79 | 80 | static gko_pool *getInstance(); 81 | static void setFuncTable(char(*cmd_list)[CMD_LEN], func_t * func_list, 82 | int cmdcount); 83 | 84 | int getPort() const; 85 | void setPort(int port); 86 | s_option_t *getOption() const; 87 | void setOption(s_option_t *option); 88 | 89 | /// close conn, shutdown && close 90 | int conn_close(); 91 | /// global run func 92 | int gko_run(); 93 | int gko_loopexit(int timeout); 94 | 95 | private: 96 | static gko_pool * _instance; 97 | /// FUNC DICT 98 | static char (* cmd_list_p)[CMD_LEN]; 99 | ///server func list 100 | static func_t * func_list_p; 101 | /// cmd type conut 102 | static int cmd_count; 103 | /// global lock 104 | static pthread_mutex_t instance_lock; 105 | 106 | int g_curr_thread; 107 | struct thread_worker ** g_worker_list; 108 | struct event_base *g_ev_base; 109 | int g_total_clients; 110 | struct conn_client **g_client_list; 111 | struct conn_server *g_server; 112 | int port; 113 | s_option_t * option; 114 | 115 | static void conn_send_data(int fd, void *str, unsigned int len); 116 | /// Accept new connection 117 | static void conn_tcp_server_accept(int fd, short ev, void *arg); 118 | /// close conn, shutdown && close 119 | static void * thread_worker_init(void *arg); 120 | /// close conn, shutdown && close 121 | static void thread_worker_process(int fd, short ev, void *arg); 122 | /// Event on data from client 123 | static void conn_tcp_server_on_data(int fd, short ev, void *arg); 124 | /// parse the request return the proper func handle num 125 | static int parse_req(char *req); 126 | 127 | int thread_worker_new(int id); 128 | int thread_list_find_next(void); 129 | int conn_client_list_init(void); 130 | int gingko_serv_async_server_base_init(void); 131 | int gingko_clnt_async_server_base_init(s_host_t * the_host); 132 | int gko_async_server_base_init(void); 133 | /// Accept new connection, start listen etc. 134 | int conn_tcp_server(struct conn_server *c); 135 | /// Accept new connection 136 | struct conn_client * add_new_conn_client(int client_fd); 137 | /// Event on data from client 138 | int conn_client_list_find_free(); 139 | /// clear client struct 140 | int conn_client_clear(struct conn_client *client); 141 | /// clear the "session" 142 | int conn_client_free(struct conn_client *client); 143 | /// Get client object from pool by given client_id 144 | struct conn_client * conn_client_list_get(int id); 145 | /// Dispatch to worker 146 | void thread_worker_dispatch(int sig_id); 147 | /// init the whole thread pool 148 | int thread_init(); 149 | /// construct func 150 | gko_pool(const int pt); 151 | /// another construct func 152 | gko_pool(); 153 | }; 154 | 155 | #endif /** ASYNC_POOL_H_ **/ 156 | -------------------------------------------------------------------------------- /src/async_threads.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * async_threads.cpp 3 | * gingko 4 | * 5 | * Created by Auxten on 11-4-16. 6 | * Copyright 2011 Baidu. All rights reserved. 7 | * 8 | **/ 9 | 10 | #include "gingko.h" 11 | #include "async_pool.h" 12 | #include "log.h" 13 | 14 | /** 15 | * @brief create new thread worker 16 | * 17 | * @see 18 | * @note 19 | * @author auxten 20 | * @date 2011-8-1 21 | **/ 22 | int gko_pool::thread_worker_new(int id) 23 | { 24 | int ret; 25 | 26 | struct thread_worker *worker = new struct thread_worker; 27 | if(! worker) 28 | { 29 | gko_log(FATAL, "new thread_worker failed"); 30 | return -1; 31 | } 32 | 33 | int fds[2]; 34 | if (pipe(fds) != 0) 35 | { 36 | gko_log(FATAL, "pipe error"); 37 | return -1; 38 | } 39 | worker->notify_recv_fd = fds[0]; 40 | worker->notify_send_fd = fds[1]; 41 | 42 | worker->ev_base = (struct event_base*)event_init(); 43 | if (!worker->ev_base) 44 | { 45 | gko_log(FATAL, "Worker event base initialize error"); 46 | return -1; 47 | } 48 | 49 | pthread_attr_t thread_attr; 50 | pthread_attr_init(&thread_attr); 51 | pthread_attr_setstacksize(&thread_attr, MYSTACKSIZE); 52 | ///pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_DETACHED); 53 | ret = pthread_create(&worker->tid, &thread_attr, thread_worker_init, 54 | (void *) worker); 55 | if (ret) 56 | { 57 | gko_log(FATAL, "Thread create error"); 58 | return -1; 59 | } 60 | worker->id = id; 61 | *(g_worker_list + id) = worker; 62 | ///gko_log(NOTICE, "thread_worker_new :%d",(*(g_worker_list+id))->notify_send_fd ); 63 | return 0; 64 | } 65 | 66 | /** 67 | * @brief Worker initialize 68 | * 69 | * @see 70 | * @note 71 | * @author auxten 72 | * @date 2011-8-1 73 | **/ 74 | void * gko_pool::thread_worker_init(void *arg) 75 | { 76 | struct thread_worker *worker = (struct thread_worker *) arg; 77 | event_set(&worker->ev_notify, worker->notify_recv_fd, EV_READ | EV_PERSIST, 78 | thread_worker_process, worker); 79 | event_base_set(worker->ev_base, &worker->ev_notify); 80 | event_add(&worker->ev_notify, 0); 81 | event_base_loop(worker->ev_base, 0); 82 | 83 | return NULL; 84 | } 85 | 86 | /** 87 | * @brief Transfer a new event to worker 88 | * 89 | * @see 90 | * @note 91 | * @author auxten 92 | * @date 2011-8-1 93 | **/ 94 | void gko_pool::thread_worker_process(int fd, short ev, void *arg) 95 | { 96 | struct thread_worker *worker = (struct thread_worker *) arg; 97 | int c_id; 98 | read(fd, &c_id, sizeof(int)); 99 | struct conn_client *client = gko_pool::getInstance()->conn_client_list_get(c_id); 100 | 101 | if (client->client_fd) 102 | { 103 | event_set(&client->ev_read, client->client_fd, EV_READ | EV_PERSIST, 104 | conn_tcp_server_on_data, (void *) client); 105 | event_base_set(worker->ev_base, &client->ev_read); 106 | if (-1 == event_add(&client->ev_read, 0)) 107 | { 108 | gko_log(WARNING, "Cannot handle client's data event"); 109 | } 110 | } 111 | else 112 | { 113 | gko_log(WARNING, "conn_client_list_get error"); 114 | } 115 | return; 116 | } 117 | 118 | /** 119 | * @brief find an availiable thread, return thread index; on error 120 | * return -1 121 | * 122 | * @see 123 | * @note 124 | * @author auxten 125 | * @date 2011-8-1 126 | **/ 127 | int gko_pool::thread_list_find_next() 128 | { 129 | int i; 130 | int tmp; 131 | 132 | for (i = 0; i < option->worker_thread; i++) 133 | { 134 | tmp = (i + g_curr_thread + 1) % option->worker_thread; 135 | if (*(g_worker_list + tmp) && (*(g_worker_list + tmp))->tid) 136 | { 137 | g_curr_thread = tmp; 138 | return tmp; 139 | } 140 | } 141 | gko_log(WARNING, "thread pool full"); 142 | return -1; 143 | } 144 | 145 | /** 146 | * @brief init the whole thread pool 147 | * 148 | * @see 149 | * @note 150 | * @author auxten 151 | * @date 2011-8-1 152 | **/ 153 | int gko_pool::thread_init() 154 | { 155 | int i; 156 | g_worker_list = new struct thread_worker *[option->worker_thread]; 157 | if (! g_worker_list) 158 | { 159 | gko_log(FATAL, "new new struct thread_worker *[option->worker_thread] failed"); 160 | return -1; 161 | } 162 | memset(g_worker_list, 0, sizeof(struct thread_worker *) * option->worker_thread); 163 | for (i = 0; i < option->worker_thread; i++) 164 | { 165 | if(thread_worker_new(i) != 0) 166 | { 167 | gko_log(FATAL, FLF("thread_worker_new error")); 168 | return -1; 169 | } 170 | } 171 | 172 | return 0; 173 | } 174 | 175 | /** 176 | * @brief parse the request return the proper func handle num 177 | * 178 | * @see 179 | * @note 180 | * @author auxten 181 | * @date 2011-8-1 182 | **/ 183 | int gko_pool::parse_req(char *req) 184 | { 185 | int i; 186 | if (UNLIKELY(!req)) 187 | { 188 | return cmd_count - 1; 189 | } 190 | for (i = 0; i < cmd_count - 1; i++) 191 | { 192 | if (cmd_list_p[i][0] == req[0] && //todo use int 193 | cmd_list_p[i][1] == req[1] && 194 | cmd_list_p[i][2] == req[2] && 195 | cmd_list_p[i][3] == req[3]) 196 | { 197 | break; 198 | } 199 | } 200 | return i; 201 | } 202 | 203 | 204 | /** 205 | * @brief Dispatch to worker 206 | * 207 | * @see 208 | * @note 209 | * @author auxten 210 | * @date 2011-8-1 211 | **/ 212 | void gko_pool::thread_worker_dispatch(int c_id) 213 | { 214 | int worker_id; 215 | int res; 216 | worker_id = thread_list_find_next(); 217 | if (worker_id < 0) 218 | { 219 | gko_log(WARNING, "can't find available thread"); 220 | return; 221 | } 222 | res = write((*(g_worker_list + worker_id))->notify_send_fd, &c_id, 223 | sizeof(int)); 224 | if (res == -1) 225 | { 226 | gko_log(WARNING, "Pipe write error"); 227 | } 228 | } 229 | 230 | int gko_pool::gko_loopexit(int timeout) 231 | { 232 | struct timeval timev; 233 | 234 | timev.tv_sec = timeout; 235 | timev.tv_usec = 0; 236 | event_base_loopexit(g_ev_base, (timeout ? &timev : NULL)); 237 | return 0; 238 | } 239 | 240 | 241 | -------------------------------------------------------------------------------- /src/autogen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | if [ -x "`which autoreconf 2>/dev/null`" ] ; then 3 | exec autoreconf --verbose --force -i 4 | fi 5 | 6 | LIBTOOLIZE=libtoolize 7 | SYSNAME=`uname` 8 | if [ "x$SYSNAME" = "xDarwin" ] ; then 9 | LIBTOOLIZE=glibtoolize 10 | fi 11 | aclocal && \ 12 | autoheader && \ 13 | autoconf && \ 14 | automake --add-missing --force-missing --copy 15 | -------------------------------------------------------------------------------- /src/clnt_unittest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * clnt_unittest.cpp 3 | * 4 | * Created on: 2011-8-15 5 | * Author: auxten 6 | */ 7 | 8 | #define UNITTEST 9 | #ifndef GINGKO_CLNT 10 | #define GINGKO_CLNT 11 | #endif /** GINGKO_CLNT **/ 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #ifdef __APPLE__ 37 | #include 38 | #else 39 | #include 40 | #endif /** __APPLE__ **/ 41 | 42 | #include "gingko.h" 43 | #include "async_pool.h" 44 | #include "hash/xor_hash.h" 45 | #include "path.h" 46 | #include "route.h" 47 | #include "log.h" 48 | #include "snap.h" 49 | #include "option.h" 50 | #include "socket.h" 51 | #include "limit.h" 52 | #include "job_state.h" 53 | #include "gingko_clnt.h" 54 | 55 | 56 | /************** PTHREAD STUFF **************/ 57 | ///default pthread_attr_t 58 | pthread_attr_t g_attr; 59 | ///client wide lock 60 | pthread_mutex_t g_clnt_lock; 61 | ///block host set lock 62 | pthread_mutex_t g_blk_hostset_lock; 63 | ///mutex for gko.hosts_new_noready 64 | pthread_mutex_t g_hosts_new_noready_mutex; 65 | ///mutex for gko.hosts_del_noready 66 | pthread_mutex_t g_hosts_del_noready_mutex; 67 | /************** PTHREAD STUFF **************/ 68 | 69 | /// the g_job assoiate with the client 70 | s_job_t g_job; 71 | 72 | /// gingko global stuff 73 | s_gingko_global_t gko; 74 | 75 | #ifdef UNITTEST 76 | 77 | #include "gtest/gtest.h" 78 | #include "unittest.h" 79 | 80 | /************** FUNC DICT **************/ 81 | #include "gingko_common.h" 82 | /************** FUNC DICT **************/ 83 | 84 | GINGKO_OVERLOAD_S_HOST_LT 85 | 86 | 87 | int clnt_unittest_init() 88 | { 89 | memset(&g_job, 0, sizeof(g_job)); 90 | memset(&gko, 0, sizeof(gko)); 91 | memset(&gko.the_serv, 0, sizeof(gko.the_serv)); 92 | 93 | gko.opt.limit_up_rate = CLNT_LIMIT_UP_RATE; 94 | gko.opt.limit_down_rate = CLNT_LIMIT_DOWN_RATE; 95 | gko.opt.worker_thread = CLNT_ASYNC_THREAD_NUM; 96 | gko.opt.connlimit = CLNT_POOL_SIZE; 97 | gko.opt.bind_ip = htons(INADDR_ANY); 98 | gko.the_serv.port = SERV_PORT; 99 | gko.ready_to_serv = 0; 100 | gko.snap_fd = -2; 101 | strncpy(gko.opt.logpath, CLIENT_LOG, sizeof(gko.opt.logpath)); 102 | 103 | umask(0); 104 | 105 | return 0; 106 | } 107 | 108 | 109 | /// path.cpp 110 | /** 111 | * @brief ../path//// TO ../path 112 | * @brief ../path/ TO ../path 113 | * @brief ../path TO ../path 114 | * 115 | * @see 116 | * @note 117 | * @author auxten 118 | * @date 2011-8-1 119 | **/ 120 | TEST(inplace_strip_tailing_slash, with1slash) 121 | { 122 | char test_path1[MAX_PATH_LEN]; 123 | strncpy(test_path1, "aaa/", MAX_PATH_LEN); 124 | char * test_rslt = "aaa"; 125 | inplace_strip_tailing_slash(test_path1); 126 | EXPECT_STREQ(test_rslt, test_path1); 127 | } 128 | 129 | TEST(inplace_strip_tailing_slash, withmoreslash) 130 | { 131 | char test_path1[MAX_PATH_LEN]; 132 | strncpy(test_path1, "aaa////", MAX_PATH_LEN); 133 | char * test_rslt = "aaa"; 134 | inplace_strip_tailing_slash(test_path1); 135 | EXPECT_STREQ(test_rslt, test_path1); 136 | } 137 | 138 | TEST(inplace_strip_tailing_slash, withnoslash) 139 | { 140 | char test_path1[MAX_PATH_LEN]; 141 | strncpy(test_path1, "aaa", MAX_PATH_LEN); 142 | char * test_rslt = "aaa"; 143 | inplace_strip_tailing_slash(test_path1); 144 | EXPECT_STREQ(test_rslt, test_path1); 145 | } 146 | 147 | /** 148 | * @brief ../path/// TO ../path/ 149 | * @brief ../path/ TO ../path/ 150 | * @brief ../path TO ../path/ 151 | * 152 | * @see 153 | * @note 154 | * @author auxten 155 | * @date 2011-8-1 156 | **/ 157 | TEST(inplace_add_tailing_slash, with1slash) 158 | { 159 | char test_path1[MAX_PATH_LEN]; 160 | strncpy(test_path1, "aaa/", MAX_PATH_LEN); 161 | char * test_rslt = "aaa/"; 162 | inplace_add_tailing_slash(test_path1); 163 | EXPECT_STREQ(test_rslt, test_path1); 164 | } 165 | 166 | TEST(inplace_add_tailing_slash, withmoreslash) 167 | { 168 | char test_path1[MAX_PATH_LEN]; 169 | strncpy(test_path1, "aaa////", MAX_PATH_LEN); 170 | char * test_rslt = "aaa/"; 171 | inplace_add_tailing_slash(test_path1); 172 | EXPECT_STREQ(test_rslt, test_path1); 173 | } 174 | 175 | TEST(inplace_add_tailing_slash, withnoslash) 176 | { 177 | char test_path1[MAX_PATH_LEN]; 178 | strncpy(test_path1, "aaa", MAX_PATH_LEN); 179 | char * test_rslt = "aaa/"; 180 | inplace_add_tailing_slash(test_path1); 181 | EXPECT_STREQ(test_rslt, test_path1); 182 | } 183 | 184 | /** 185 | * @brief get the base name of a string, 186 | * @brief if out not NULL, cp the base path to out 187 | * @brief return the base path len 188 | * 189 | * @see 190 | * @note 191 | * for example: 192 | * /home/work/opdir -> 11 193 | * ^ 194 | * ./dir -> 2 195 | * ^ 196 | * dir -> 0 197 | * ^ 198 | * ../file -> 3 199 | * ^ 200 | * @author auxten 201 | * @date 2011-8-1 202 | **/ 203 | TEST(get_base_name_index, 1) 204 | { 205 | char test_path1[MAX_PATH_LEN]; 206 | 207 | EXPECT_EQ(get_base_name_index(test_path1, "/home/work/opdir"), 11); 208 | EXPECT_STREQ("opdir", test_path1); 209 | } 210 | 211 | TEST(get_base_name_index, 2) 212 | { 213 | char test_path1[MAX_PATH_LEN]; 214 | 215 | EXPECT_EQ(get_base_name_index(test_path1, "./dir"), 2); 216 | EXPECT_STREQ("dir", test_path1); 217 | } 218 | 219 | TEST(get_base_name_index, 3) 220 | { 221 | char test_path1[MAX_PATH_LEN]; 222 | 223 | EXPECT_EQ(get_base_name_index(test_path1, "dir"), 0); 224 | EXPECT_STREQ("dir", test_path1); 225 | } 226 | 227 | TEST(get_base_name_index, 4) 228 | { 229 | char test_path1[MAX_PATH_LEN]; 230 | 231 | EXPECT_EQ(get_base_name_index(test_path1, "../file"), 3); 232 | EXPECT_STREQ("file", test_path1); 233 | } 234 | 235 | TEST(get_base_name_index, 5) 236 | { 237 | char test_path1[MAX_PATH_LEN]; 238 | 239 | EXPECT_EQ(get_base_name_index(test_path1, "../file/"), 8); 240 | EXPECT_STREQ("", test_path1); 241 | } 242 | 243 | /** 244 | * @brief in : const char * dir_name, const char * base_name 245 | * @brief out : dir_name/base_name 246 | * 247 | * @see 248 | * @note 249 | * @author auxten 250 | * @date 2011-8-1 251 | **/ 252 | TEST(merge_path, normal) 253 | { 254 | char result[MAX_PATH_LEN]; 255 | EXPECT_EQ(0, merge_path(result, "dirname", "basename")); 256 | EXPECT_STREQ("dirname/basename", result); 257 | } 258 | 259 | TEST(merge_path, dirwithslash) 260 | { 261 | char result[MAX_PATH_LEN]; 262 | EXPECT_EQ(0, merge_path(result, "dirname/", "basename")); 263 | EXPECT_STREQ("dirname/basename", result); 264 | } 265 | 266 | TEST(merge_path, dirwith2slash) 267 | { 268 | char result[MAX_PATH_LEN]; 269 | EXPECT_EQ(0, merge_path(result, "dirname//", "basename")); 270 | EXPECT_STREQ("dirname/basename", result); 271 | } 272 | 273 | /** 274 | * @brief change remote path to local path, store it in path 275 | * 276 | * @see 277 | * @note 278 | * path: ../test/.DS_Store 279 | * req_path: ../test 280 | * local_path: ../output2/ 281 | * if (dst_path_exist) 282 | * path output: ../output2/test/.DS_Store 283 | * else 284 | * path output: ../output2/.DS_Store 285 | * @author auxten 286 | * @date 2011-8-1 287 | **/ 288 | TEST(change_to_local_path, dst_path_exist) 289 | { 290 | char path[MAX_PATH_LEN]; 291 | strncpy(path, "../test/.DS_Store", MAX_PATH_LEN); 292 | EXPECT_EQ(0, change_to_local_path(path, "../test", "../output2/", 1)); 293 | EXPECT_STREQ("../output2/test/.DS_Store", path); 294 | } 295 | 296 | TEST(change_to_local_path, dst_path_nonexist) 297 | { 298 | char path[MAX_PATH_LEN]; 299 | strncpy(path, "../test/.DS_Store", MAX_PATH_LEN); 300 | EXPECT_EQ(0, change_to_local_path(path, "../test", "../output2/", 0)); 301 | EXPECT_STREQ("../output2/.DS_Store", path); 302 | } 303 | 304 | /** 305 | * @brief change current working dir path to absolute path 306 | * 307 | * @see 308 | * @note 309 | * result is stored in abs_path 310 | * return abs_path on succeed else NULL 311 | * @author auxten 312 | * @date 2011-8-1 313 | **/ 314 | TEST(cwd_path_to_abs_path, relative_path) 315 | { 316 | char path[MAX_PATH_LEN]; 317 | char orig_path[MAX_PATH_LEN]; 318 | EXPECT_NE((char *)NULL, getcwd(orig_path, MAX_PATH_LEN)); 319 | EXPECT_EQ(0, chdir("/home")); 320 | EXPECT_NE((char *)NULL, cwd_path_to_abs_path(path, "work")); 321 | EXPECT_STREQ("/home/work", path); 322 | EXPECT_EQ(0, chdir(orig_path)); 323 | } 324 | 325 | TEST(cwd_path_to_abs_path, abs_path) 326 | { 327 | char path[MAX_PATH_LEN]; 328 | EXPECT_NE((char *)NULL, cwd_path_to_abs_path(path, "/work")); 329 | EXPECT_STREQ("/work", path); 330 | } 331 | 332 | /** 333 | * @brief get the symlink dest's absolute path, store it in abs_path 334 | * 335 | * @see 336 | * @note 337 | * @author auxten 338 | * @date 2011-8-1 339 | **/ 340 | TEST(symlink_dest_to_abs_path, symlink_to_relative_path) 341 | { 342 | char path[MAX_PATH_LEN]; 343 | char path2[MAX_PATH_LEN]; 344 | EXPECT_NE((char *)NULL, getcwd(path2, MAX_PATH_LEN)); 345 | strncat(path2, "/../testcase/test.sh", MAX_PATH_LEN); 346 | EXPECT_NE((char *)NULL, symlink_dest_to_abs_path(path, "../testcase/test.sh.ln")); 347 | EXPECT_STREQ(path2, path); 348 | } 349 | 350 | TEST(symlink_dest_to_abs_path, symlink_to_abs_path) 351 | { 352 | char path[MAX_PATH_LEN]; 353 | EXPECT_NE((char *)NULL, symlink_dest_to_abs_path(path, "../testcase/test.sh.absln")); 354 | EXPECT_STREQ("/home/auxten", path); 355 | } 356 | 357 | ///socket.cpp 358 | /** 359 | * @brief connect to a host 360 | * 361 | * @see 362 | * @note 363 | * h: pointer to s_host_t 364 | * recv_sec: receive timeout seconds, 0 for never timeout 365 | * return the socket when succ 366 | * return < 0 when error, specially HOST_DOWN_FAIL indicate host dead 367 | * @author auxten 368 | * @date 2011-8-1 369 | **/ 370 | TEST(socket, Connect_host_Setnonblock_Setblock_Close_socket) 371 | { 372 | int fd, flags; 373 | s_host_t h = {"localhost", 22}; 374 | 375 | ///establish socket 376 | fd = connect_host(&h, 5, 5); 377 | EXPECT_LE(0, fd); 378 | 379 | ///setnonblock 380 | EXPECT_EQ(0, setnonblock(fd)); 381 | flags = fcntl(fd, F_GETFL); 382 | EXPECT_LE(0, flags); 383 | EXPECT_NE(0, flags & O_NONBLOCK); 384 | 385 | ///try setnonblock again 386 | EXPECT_EQ(0, setnonblock(fd)); 387 | flags = fcntl(fd, F_GETFL); 388 | EXPECT_LE(0, flags); 389 | EXPECT_NE(0, flags & O_NONBLOCK); 390 | 391 | ///setblock 392 | EXPECT_EQ(0, setblock(fd)); 393 | flags = fcntl(fd, F_GETFL); 394 | EXPECT_LE(0, flags); 395 | EXPECT_EQ(0, flags & O_NONBLOCK); 396 | 397 | ///try setblock again 398 | EXPECT_EQ(0, setblock(fd)); 399 | flags = fcntl(fd, F_GETFL); 400 | EXPECT_LE(0, flags); 401 | EXPECT_EQ(0, flags & O_NONBLOCK); 402 | 403 | ///close socket 404 | EXPECT_EQ(0, close_socket(fd)); 405 | } 406 | 407 | 408 | ///options.cpp 409 | /** 410 | * @brief process args for client 411 | * 412 | * @see 413 | * @note 414 | * @author auxten 415 | * @date 2011-8-1 416 | **/ 417 | //int clnt_parse_opt(int argc, char *argv[], s_job_t * jo) 418 | TEST(clnt_parse_opt, clnt_parse_opt) 419 | { 420 | const int max_argc = 30; 421 | char argv_test_buf[max_argc][MAX_PATH_LEN] = 422 | { 423 | "gkocp", 424 | "-l", 425 | "./clnt.log", 426 | "127.0.0.1:./src", 427 | "./dest", 428 | "-o", "-c", "-u", "17", "-d", "18", "-r", "19", "-w", "20", 429 | "-t", "21", "-n", "22", "-s", "23", "-b", "127.0.0.1", "-p", "2121", "--debug" 430 | }; 431 | char * argv_test[max_argc]; 432 | for (int i = 0; i < max_argc; i++) 433 | { 434 | argv_test[i] = (char *)argv_test_buf[i]; 435 | } 436 | int argc_test = 26; 437 | EXPECT_EQ(0, clnt_parse_opt(argc_test, argv_test, &g_job)); 438 | //EXPECT_EQ(htonl(INADDR_LOOPBACK), gko.opt.bind_ip); 439 | EXPECT_EQ(22, gko.opt.connlimit); 440 | EXPECT_EQ(17*1024*1024, gko.opt.limit_up_rate); 441 | EXPECT_EQ(18*1024*1024, gko.opt.limit_down_rate); 442 | EXPECT_EQ(19*1024*1024, gko.opt.limit_disk_r_rate); 443 | EXPECT_EQ(20*1024*1024, gko.opt.limit_disk_w_rate); 444 | EXPECT_EQ(0, gko.opt.need_help); 445 | EXPECT_EQ(1, gko.opt.need_progress); 446 | EXPECT_EQ(2121, gko.the_serv.port); 447 | EXPECT_EQ(23, gko.opt.seed_time); 448 | EXPECT_EQ(1, gko.opt.to_continue); 449 | EXPECT_EQ(1, gko.opt.to_debug); 450 | EXPECT_EQ(21, gko.opt.worker_thread); 451 | EXPECT_STREQ("./clnt.log", gko.opt.logpath); 452 | 453 | } 454 | 455 | TEST(clnt_show_version, clnt_show_version) 456 | { 457 | EXPECT_NO_FATAL_FAILURE(clnt_show_version()); 458 | } 459 | 460 | TEST(clnt_show_help, clnt_show_help) 461 | { 462 | EXPECT_NO_FATAL_FAILURE(clnt_show_help()); 463 | } 464 | 465 | int main(int argc, char *argv[]) 466 | { 467 | if(clnt_unittest_init() != 0) 468 | { 469 | perror("clnt_unittest_init failed"); 470 | exit(1); 471 | } 472 | ::testing::InitGoogleTest(&argc, argv); 473 | 474 | return RUN_ALL_TESTS(); 475 | } 476 | #else 477 | int main() 478 | { 479 | 480 | } 481 | #endif /* UNITTEST */ 482 | -------------------------------------------------------------------------------- /src/configure.in: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([1.68]) 5 | AC_INIT([libxor_hash.a,gingko_serv,gingko_clnt,unittest], [3.14], [blog.optool.net]) 6 | AC_CONFIG_SRCDIR([config.h.in]) 7 | AM_CONFIG_HEADER([config.h]) 8 | #AC_CONFIG_HEADERS([config.h]) 9 | AM_INIT_AUTOMAKE([libxor_hash.a,gingko_serv,gingko_clnt,unittest],3.14) 10 | 11 | ########################################################################## 12 | # debug compilation support 13 | ########################################################################## 14 | 15 | AC_MSG_CHECKING([whether to build with debug information]) 16 | AC_ARG_ENABLE([debug], 17 | [AS_HELP_STRING([--enable-debug], 18 | [enable debug data generation (def=no)])], 19 | [debugit="$enableval"], 20 | [debugit=no]) 21 | AC_MSG_RESULT([$debugit]) 22 | 23 | if test x"$debugit" = x"yes"; then 24 | AC_DEFINE([GKO_DEBUG],[],[Debug Mode]) 25 | AM_CXXFLAGS="-ggdb -Wall -O0" 26 | else 27 | AC_DEFINE([NGKO_DEBUG],[],[No-debug Mode]) 28 | AM_CXXFLAGS="$AM_CXXFLAGS" 29 | fi 30 | 31 | AC_SUBST([AM_CXXFLAGS]) 32 | ########################################################################## 33 | 34 | ########################################################################## 35 | # unittest compilation support 36 | ########################################################################## 37 | 38 | AC_MSG_CHECKING([whether to build unittest version]) 39 | AC_ARG_ENABLE([unittest], 40 | [AS_HELP_STRING([--enable-unittest], 41 | [enable unittest code generation (def=no)])], 42 | [unittestit="$enableval"], 43 | [unittestit=no]) 44 | AC_MSG_RESULT([$unittestit]) 45 | 46 | if test x"$unittestit" = x"yes"; then 47 | AC_DEFINE([UNITTEST],[],[Unittest Mode]) 48 | AM_CXXFLAGS="-ggdb -Wall" 49 | else 50 | AC_DEFINE([NUNITTEST],[],[No-unittest Mode]) 51 | AM_CXXFLAGS="$AM_CXXFLAGS" 52 | fi 53 | 54 | AC_SUBST([AM_CXXFLAGS]) 55 | ########################################################################## 56 | 57 | ########################################################################## 58 | # profile compilation support 59 | ########################################################################## 60 | 61 | AC_MSG_CHECKING([whether to build with profile information]) 62 | AC_ARG_ENABLE([profile], 63 | [AS_HELP_STRING([--enable-profile], 64 | [enable profile code generation (def=no)])], 65 | [profileit="$enableval"], 66 | [profileit=no]) 67 | AC_MSG_RESULT([$profileit]) 68 | 69 | if test x"$profileit" = x"yes"; then 70 | AC_DEFINE([PROFILE],[],[Profile Mode]) 71 | AM_CXXFLAGS="-ggdb -Wall -lprofiler" 72 | else 73 | AC_DEFINE([NPROFILE],[],[No-profile Mode]) 74 | AM_CXXFLAGS="$AM_CXXFLAGS" 75 | fi 76 | 77 | AC_SUBST([AM_CXXFLAGS]) 78 | ########################################################################## 79 | 80 | ########################################################################## 81 | # gprofile compilation support 82 | ########################################################################## 83 | 84 | AC_MSG_CHECKING([whether to build with gprofile information]) 85 | AC_ARG_ENABLE([gprofile], 86 | [AS_HELP_STRING([--enable-gprofile], 87 | [enable gprofile code generation (def=no)])], 88 | [gprofileit="$enableval"], 89 | [gprofileit=no]) 90 | AC_MSG_RESULT([$gprofileit]) 91 | 92 | if test x"$gprofileit" = x"yes"; then 93 | AC_DEFINE([GPROFILE],[],[Profile Mode]) 94 | AM_CXXFLAGS="-ggdb -Wall -pg" 95 | else 96 | AC_DEFINE([NGPROFILE],[],[No-gprofile Mode]) 97 | AM_CXXFLAGS="$AM_CXXFLAGS" 98 | fi 99 | 100 | AC_SUBST([AM_CXXFLAGS]) 101 | ########################################################################## 102 | 103 | ########################################################################## 104 | # O3 optimize compilation support 105 | ########################################################################## 106 | 107 | AC_MSG_CHECKING([whether to build with O3 optimization]) 108 | AC_ARG_ENABLE([o3], 109 | [AS_HELP_STRING([--enable-o3], 110 | [enable O3 code generation (def=no)])], 111 | [o3it="$enableval"], 112 | [o3it=no]) 113 | AC_MSG_RESULT([$o3it]) 114 | 115 | if test x"$o3it" = x"yes"; then 116 | AC_DEFINE([O3],[],[O3 Mode]) 117 | AM_CXXFLAGS="-Wall -O3 -march=nocona" 118 | else 119 | AC_DEFINE([NO3],[],[No-O3 Mode]) 120 | AM_CXXFLAGS="$AM_CXXFLAGS" 121 | fi 122 | 123 | AC_SUBST([AM_CXXFLAGS]) 124 | ########################################################################## 125 | 126 | # Checks for programs. 127 | AC_PROG_CXX 128 | AC_PROG_CC 129 | AC_PROG_MAKE_SET 130 | 131 | # Checks for libraries. 132 | AC_SEARCH_LIBS([pthread_rwlock_init], [pthread]) 133 | AC_SEARCH_LIBS([clock_gettime], [rt]) 134 | AC_PROG_RANLIB 135 | 136 | # Checks for header files. 137 | AC_CHECK_HEADERS([arpa/inet.h fcntl.h netdb.h netinet/in.h stdlib.h string.h sys/socket.h unistd.h]) 138 | 139 | # Checks for typedefs, structures, and compiler characteristics. 140 | AC_TYPE_MODE_T 141 | AC_TYPE_SIZE_T 142 | 143 | # Checks for library functions. 144 | AC_FUNC_FORK 145 | AC_CHECK_FUNCS([gethostbyname memset select socket strerror gettimeofday inet_ntoa poll]) 146 | 147 | AC_CONFIG_SUBDIRS([hash]) 148 | AC_CONFIG_FILES([Makefile hash/Makefile]) 149 | AC_OUTPUT 150 | -------------------------------------------------------------------------------- /src/erase_job.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import telnetlib 3 | import sys 4 | 5 | tn = telnetlib.Telnet('localhost', 2120) 6 | tn.write('ERSE\t%s'% sys.argv[1]) 7 | print tn.read_all() 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /src/gingko_clnt.h: -------------------------------------------------------------------------------- 1 | /* 2 | * gingko_clnt.h 3 | * 4 | * Created on: 2011-8-15 5 | * Author: auxten 6 | */ 7 | 8 | #ifndef GINGKO_CLNT_H_ 9 | #define GINGKO_CLNT_H_ 10 | 11 | /// CORE algrithm: hash the host to the data ring 12 | s_host_hash_result_t * host_hash(s_job_t * jo, const s_host_t * new_host, 13 | s_host_hash_result_t * result, const u_char usage); 14 | 15 | /// sent GET handler 16 | GKO_INT64 get_blocks_c(s_job_t * jo, s_host_t * dhost, GKO_INT64 num, 17 | GKO_INT64 count, 18 | u_char flag, char * buf); 19 | 20 | /// send HELO handler 21 | int helo_serv_c(void * arg, int fd, s_host_t * server); 22 | 23 | /// send JOIN handler 24 | void * join_job_c(void * arg, int fd); 25 | 26 | /// send QUIT handler 27 | int quit_job_c(const s_host_t * quit_host, const s_host_t * server, const char * uri); 28 | 29 | #endif /* GINGKO_CLNT_H_ */ 30 | -------------------------------------------------------------------------------- /src/gingko_serv.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * gingko_serv.cpp 3 | * gingko 4 | * 5 | * Created by Auxten on 11-4-9. 6 | * Copyright 2011 Baidu. All rights reserved. 7 | * 8 | **/ 9 | 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #ifdef __APPLE__ 33 | #include 34 | #elif defined (__FreeBSD__) 35 | #include 36 | #include 37 | #include 38 | #else 39 | #include 40 | #endif /** __APPLE__ **/ 41 | 42 | #include 43 | #include 44 | #include 45 | #include 46 | 47 | #include "gingko.h" 48 | #include "async_pool.h" 49 | #include "hash/xor_hash.h" 50 | #include "path.h" 51 | #include "log.h" 52 | #include "seed.h" 53 | #include "socket.h" 54 | #include "option.h" 55 | #include "job_state.h" 56 | 57 | 58 | /************** PTHREAD STUFF **************/ 59 | ///server wide lock 60 | extern pthread_mutex_t g_grand_lock; 61 | ///job specific lock 62 | extern s_lock_t g_job_lock[MAX_JOBS]; 63 | /************** PTHREAD STUFF **************/ 64 | 65 | extern std::map g_m_jobs; 66 | 67 | 68 | /** 69 | * @brief erase job related stuff only for server 70 | * 71 | * @see s_job_t struct 72 | * @note 73 | * @author auxten 74 | * @date 2011-8-1 75 | **/ 76 | int erase_job(std::string &uri_string) 77 | { 78 | int ret; 79 | s_job_t *jo; 80 | GKO_INT64 progress; 81 | GKO_INT64 percent; 82 | 83 | /// jobs map 84 | std::map::iterator it; 85 | 86 | { 87 | pthread_mutex_lock(&g_grand_lock); 88 | if ((it = g_m_jobs.find(uri_string)) == g_m_jobs.end()) 89 | { 90 | ret = -1; 91 | gko_log(FATAL, "erase %s fail", uri_string.c_str()); 92 | pthread_mutex_unlock(&g_grand_lock); 93 | goto ERASE_RET; 94 | } 95 | jo = it->second; 96 | jo->job_state = JOB_TO_BE_ERASED; 97 | /** then erase job for the job map **/ 98 | g_m_jobs.erase(uri_string); 99 | gko_log(NOTICE, "g_m_jobs.size: %lld", (GKO_UINT64) g_m_jobs.size()); 100 | pthread_mutex_unlock(&g_grand_lock); 101 | } 102 | /** cancel the hash threads if any progress < 99% **/ 103 | progress = array_sum(jo->hash_progress, XOR_HASH_TNUM); 104 | percent = jo->total_size ? progress * 100 / jo->total_size : 100; 105 | 106 | if (percent < 100) 107 | { 108 | for (int i = 0; i < XOR_HASH_TNUM; i++) 109 | { 110 | if (! pthread_cancel(jo->hash_worker[i])) 111 | { 112 | gko_log(NOTICE, "hash thread %d canceling", i); 113 | } 114 | else 115 | { 116 | gko_log(FATAL, "hash thread %d cancel failed", i); 117 | } 118 | } 119 | } 120 | 121 | /** 122 | * sleep for about write timeout time + 2 123 | * we must wait for the send thread call write 124 | * timeout and stop sending blocks. then we 125 | * can erase the job successfully 126 | **/ 127 | sleep(ERASE_JOB_MEM_WAIT); 128 | 129 | 130 | /** clean the job struct **/ 131 | pthread_mutex_lock(&g_job_lock[jo->lock_id].lock); 132 | if (jo->blocks && jo->block_count) 133 | { 134 | delete [](jo->blocks); 135 | jo->blocks = NULL; 136 | } 137 | if (jo->files && jo->file_count) 138 | { 139 | delete [](jo->files); 140 | jo->files = NULL; 141 | } 142 | if (jo->host_set) 143 | { 144 | delete jo->host_set; 145 | jo->host_set = NULL; 146 | } 147 | for (int i = 0; i < XOR_HASH_TNUM; i++) 148 | { 149 | if (jo->hash_buf[i]) 150 | { 151 | delete [](jo->hash_buf[i]); 152 | jo->hash_buf[i] = NULL; 153 | } 154 | } 155 | pthread_mutex_unlock(&g_job_lock[jo->lock_id].lock); 156 | 157 | /** for safety re-init the rwlock **/ 158 | pthread_mutex_destroy(&(g_job_lock[jo->lock_id].lock)); 159 | pthread_mutex_init(&(g_job_lock[jo->lock_id].lock), NULL); 160 | g_job_lock[jo->lock_id].state = LK_FREE; 161 | 162 | delete jo; 163 | gko_log(NOTICE, "job '%s' erased", uri_string.c_str()); 164 | ret = 0; 165 | 166 | ERASE_RET: 167 | return ret; 168 | } 169 | 170 | -------------------------------------------------------------------------------- /src/gingko_serv.h: -------------------------------------------------------------------------------- 1 | /* 2 | * gingko_serv.h 3 | * 4 | * Created on: 2011-8-15 5 | * Author: auxten 6 | */ 7 | 8 | #ifndef GINGKO_SERV_H_ 9 | #define GINGKO_SERV_H_ 10 | 11 | #include 12 | 13 | 14 | /** 15 | * @brief erase job related stuff only for server 16 | * 17 | * @see s_job_t struct 18 | * @note 19 | * @author auxten 20 | * @date 2011-8-1 21 | **/ 22 | int erase_job(std::string &uri_string); 23 | 24 | #endif /* GINGKO_SERV_H_ */ 25 | -------------------------------------------------------------------------------- /src/hash/AUTHORS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auxten/gingko/ceeeaaa9bb4ee7e6cde43f45f80f4e7c71f97ff2/src/hash/AUTHORS -------------------------------------------------------------------------------- /src/hash/ChangeLog: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auxten/gingko/ceeeaaa9bb4ee7e6cde43f45f80f4e7c71f97ff2/src/hash/ChangeLog -------------------------------------------------------------------------------- /src/hash/Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = . 2 | INCLUDES = -I$(srcdir)/src \ 3 | -I$(srcdir)/../../lib/libev/include 4 | # -I$(srcdir)/../../../../../../third-64/libev/include 5 | # -I$(srcdir)/../../../../../../../third-64/libev/include 6 | #export INCLUDES 7 | 8 | noinst_LIBRARIES=libxor_hash.a libmd5.a libgko_zip.a 9 | 10 | DEFS+=-O0 -D_GNU_SOURCE -ggdb -funsigned-char 11 | libxor_hash_a_SOURCES=xor_hash.cpp 12 | libxor_hash_a_LDFLAGS=-O0 -D_GNU_SOURCE -ggdb -funsigned-char 13 | 14 | DEFS+=-O2 -D_GNU_SOURCE -ggdb 15 | libmd5_a_SOURCES=md5.cpp 16 | libmd5_a_LDFLAGS=-O2 -D_GNU_SOURCE -ggdb 17 | 18 | DEFS+=-D_GNU_SOURCE -ggdb -O3 -I. -Wall -W 19 | libgko_zip_a_SOURCES=gko_zip.cpp lz4.cpp 20 | libgko_zip_a_LDFLAGS=-D_GNU_SOURCE -ggdb -O3 -I. -Wall -W -------------------------------------------------------------------------------- /src/hash/NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auxten/gingko/ceeeaaa9bb4ee7e6cde43f45f80f4e7c71f97ff2/src/hash/NEWS -------------------------------------------------------------------------------- /src/hash/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/auxten/gingko/ceeeaaa9bb4ee7e6cde43f45f80f4e7c71f97ff2/src/hash/README -------------------------------------------------------------------------------- /src/hash/configure.in: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([1.68]) 5 | AC_INIT([libxor_hash.a], [3.14], [blog.auxten.com]) 6 | AC_CONFIG_SRCDIR([config.h.in]) 7 | AM_CONFIG_HEADER([config.h]) 8 | #AC_CONFIG_HEADERS([config.h]) 9 | AM_INIT_AUTOMAKE([libxor_hash.a],3.14) 10 | 11 | ########################################################################## 12 | # debug compilation support 13 | ########################################################################## 14 | 15 | AC_MSG_CHECKING([whether to build with debug information]) 16 | AC_ARG_ENABLE([debug], 17 | [AS_HELP_STRING([--enable-debug], 18 | [enable debug data generation (def=no)])], 19 | [debugit="$enableval"], 20 | [debugit=no]) 21 | AC_MSG_RESULT([$debugit]) 22 | 23 | if test x"$debugit" = x"yes"; then 24 | AC_DEFINE([DEBUG],[],[Debug Mode]) 25 | AM_CXXFLAGS="-ggdb -Wall -Wno-uninitialized -O0" 26 | else 27 | AC_DEFINE([NDEBUG],[],[No-debug Mode]) 28 | AM_CXXFLAGS="$AM_CXXFLAGS" 29 | fi 30 | 31 | AC_SUBST([AM_CXXFLAGS]) 32 | ########################################################################## 33 | 34 | ########################################################################## 35 | # profile compilation support 36 | ########################################################################## 37 | 38 | AC_MSG_CHECKING([whether to build with profile information]) 39 | AC_ARG_ENABLE([profile], 40 | [AS_HELP_STRING([--enable-profile], 41 | [enable profile code generation (def=no)])], 42 | [profileit="$enableval"], 43 | [profileit=no]) 44 | AC_MSG_RESULT([$profileit]) 45 | 46 | if test x"$profileit" = x"yes"; then 47 | AC_DEFINE([PROFILE],[],[Profile Mode]) 48 | AM_CXXFLAGS="-ggdb -Wall -O2 -lprofiler" 49 | else 50 | AC_DEFINE([NPROFILE],[],[No-profile Mode]) 51 | AM_CXXFLAGS="$AM_CXXFLAGS" 52 | fi 53 | 54 | AC_SUBST([AM_CXXFLAGS]) 55 | ########################################################################## 56 | 57 | ########################################################################## 58 | # gprofile compilation support 59 | ########################################################################## 60 | 61 | AC_MSG_CHECKING([whether to build with gprofile information]) 62 | AC_ARG_ENABLE([gprofile], 63 | [AS_HELP_STRING([--enable-gprofile], 64 | [enable gprofile code generation (def=no)])], 65 | [gprofileit="$enableval"], 66 | [gprofileit=no]) 67 | AC_MSG_RESULT([$gprofileit]) 68 | 69 | if test x"$gprofileit" = x"yes"; then 70 | AC_DEFINE([GPROFILE],[],[Profile Mode]) 71 | AM_CXXFLAGS="-ggdb -Wall -pg" 72 | else 73 | AC_DEFINE([NGPROFILE],[],[No-gprofile Mode]) 74 | AM_CXXFLAGS="$AM_CXXFLAGS" 75 | fi 76 | 77 | AC_SUBST([AM_CXXFLAGS]) 78 | ########################################################################## 79 | 80 | ########################################################################## 81 | # O3 optimize compilation support 82 | ########################################################################## 83 | 84 | AC_MSG_CHECKING([whether to build with O3 optimization]) 85 | AC_ARG_ENABLE([o3], 86 | [AS_HELP_STRING([--enable-o3], 87 | [enable O3 code generation (def=no)])], 88 | [o3it="$enableval"], 89 | [o3it=no]) 90 | AC_MSG_RESULT([$o3it]) 91 | 92 | if test x"$o3it" = x"yes"; then 93 | AC_DEFINE([O3],[],[O3 Mode]) 94 | AM_CXXFLAGS="-Wall -O3 -march=nocona" 95 | else 96 | AC_DEFINE([NO3],[],[No-O3 Mode]) 97 | AM_CXXFLAGS="$AM_CXXFLAGS" 98 | fi 99 | 100 | AC_SUBST([AM_CXXFLAGS]) 101 | ########################################################################## 102 | 103 | # Checks for programs. 104 | AC_PROG_CXX 105 | AC_PROG_CC 106 | AC_PROG_MAKE_SET 107 | 108 | # Checks for libraries. 109 | AC_SEARCH_LIBS([pthread_rwlock_init], [pthread]) 110 | AC_SEARCH_LIBS([clock_gettime], [rt]) 111 | AC_PROG_RANLIB 112 | 113 | # Checks for header files. 114 | AC_CHECK_HEADERS([arpa/inet.h fcntl.h netdb.h netinet/in.h stdlib.h string.h sys/socket.h unistd.h]) 115 | 116 | # Checks for typedefs, structures, and compiler characteristics. 117 | AC_TYPE_MODE_T 118 | AC_TYPE_SIZE_T 119 | 120 | # Checks for library functions. 121 | AC_FUNC_FORK 122 | AC_CHECK_FUNCS([gethostbyname memset select socket strerror gettimeofday inet_ntoa]) 123 | 124 | AC_CONFIG_FILES([Makefile]) 125 | AC_OUTPUT 126 | -------------------------------------------------------------------------------- /src/hash/gko_zip.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * gko_zip.cpp 3 | * 4 | * Created on: Jan 11, 2012 5 | * Author: auxten 6 | */ 7 | 8 | 9 | #include "lz4.h" 10 | #include "gko_zip.h" 11 | 12 | int gko_zip(char* source, char* dest, int isize) 13 | { 14 | return LZ4_compress(source, dest, isize); 15 | } 16 | 17 | int gko_unzip(char* source, char* dest, int osize) 18 | { 19 | return LZ4_uncompress(source, dest, osize); 20 | } 21 | -------------------------------------------------------------------------------- /src/hash/gko_zip.h: -------------------------------------------------------------------------------- 1 | /* 2 | * gko_zip.h 3 | * 4 | * Created on: Jan 11, 2012 5 | * Author: auxten 6 | */ 7 | 8 | #ifndef GKO_ZIP_H_ 9 | #define GKO_ZIP_H_ 10 | 11 | int gko_zip(char* source, char* dest, int isize); 12 | int gko_unzip(char* source, char* dest, int osize); 13 | 14 | #endif /* GKO_ZIP_H_ */ 15 | -------------------------------------------------------------------------------- /src/hash/lz4.h: -------------------------------------------------------------------------------- 1 | /* 2 | LZ4 - Fast LZ compression algorithm 3 | Header File 4 | Copyright (C) 2011, Yann Collet. 5 | BSD License 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions are 9 | met: 10 | 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above 14 | copyright notice, this list of conditions and the following disclaimer 15 | in the documentation and/or other materials provided with the 16 | distribution. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | */ 30 | #pragma once 31 | 32 | #if defined (__cplusplus) 33 | extern "C" { 34 | #endif 35 | 36 | 37 | //**************************** 38 | // Simple Functions 39 | //**************************** 40 | 41 | int LZ4_compress (char* source, char* dest, int isize); 42 | int LZ4_uncompress (char* source, char* dest, int osize); 43 | 44 | /* 45 | LZ4_compress() : 46 | return : the number of bytes in compressed buffer dest 47 | note : destination buffer must be already allocated. 48 | To avoid any problem, size it to handle worst cases situations (input data not compressible) 49 | Worst case size is : "inputsize + 0.4%", with "0.4%" being at least 8 bytes. 50 | 51 | LZ4_uncompress() : 52 | osize : is the output size, therefore the original size 53 | return : the number of bytes read in the source buffer 54 | If the source stream is malformed, the function will stop decoding and return a negative result, indicating the byte position of the faulty instruction 55 | This version never writes beyond dest + osize, and is therefore protected against malicious data packets 56 | note 2 : destination buffer must be already allocated 57 | */ 58 | 59 | 60 | //**************************** 61 | // Advanced Functions 62 | //**************************** 63 | 64 | int LZ4_uncompress_unknownOutputSize (char* source, char* dest, int isize, int maxOutputSize); 65 | 66 | /* 67 | LZ4_uncompress_unknownOutputSize() : 68 | isize : is the input size, therefore the compressed size 69 | maxOutputSize : is the size of the destination buffer (which must be already allocated) 70 | return : the number of bytes decoded in the destination buffer (necessarily <= maxOutputSize) 71 | If the source stream is malformed, the function will stop decoding and return a negative result, indicating the byte position of the faulty instruction 72 | This version never writes beyond dest + maxOutputSize, and is therefore protected against malicious data packets 73 | note : This version is a bit slower than LZ4_uncompress 74 | */ 75 | 76 | 77 | int LZ4_compressCtx(void** ctx, char* source, char* dest, int isize); 78 | 79 | /* 80 | LZ4_compressCtx() : 81 | This function explicitly handles the CTX memory structure. 82 | It avoids allocating/deallocating memory between each call, improving performance when malloc is time-consuming. 83 | Note : when memory is allocated into the stack (default mode), there is no "malloc" penalty. 84 | Therefore, this function is mostly useful when memory is allocated into the heap (it requires increasing HASH_LOG value beyond STACK_LIMIT) 85 | 86 | On first call : provide a *ctx=NULL; It will be automatically allocated. 87 | On next calls : reuse the same ctx pointer. 88 | Use different pointers for different threads when doing multi-threading. 89 | 90 | note : performance difference is small, mostly noticeable in HeapMode when repetitively calling the compression function over many small segments. 91 | */ 92 | 93 | 94 | #if defined (__cplusplus) 95 | } 96 | #endif 97 | -------------------------------------------------------------------------------- /src/hash/md5.h: -------------------------------------------------------------------------------- 1 | /** 2 | * \file md5.h 3 | * 4 | * \brief MD5 message digest algorithm (hash function) 5 | * 6 | * Copyright (C) 2006-2010, Brainspark B.V. 7 | * 8 | * This file is part of PolarSSL (http://www.polarssl.org) 9 | * Lead Maintainer: Paul Bakker 10 | * 11 | * All rights reserved. 12 | * 13 | * This program is free software; you can redistribute it and/or modify 14 | * it under the terms of the GNU General Public License as published by 15 | * the Free Software Foundation; either version 2 of the License, or 16 | * (at your option) any later version. 17 | * 18 | * This program is distributed in the hope that it will be useful, 19 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 21 | * GNU General Public License for more details. 22 | * 23 | * You should have received a copy of the GNU General Public License along 24 | * with this program; if not, write to the Free Software Foundation, Inc., 25 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 26 | */ 27 | #ifndef POLARSSL_MD5_H 28 | #define POLARSSL_MD5_H 29 | 30 | #include 31 | 32 | /** 33 | * \brief MD5 context structure 34 | */ 35 | typedef struct 36 | { 37 | unsigned long total[2]; /*!< number of bytes processed */ 38 | unsigned long state[4]; /*!< intermediate digest state */ 39 | unsigned char buffer[64]; /*!< data block being processed */ 40 | 41 | unsigned char ipad[64]; /*!< HMAC: inner padding */ 42 | unsigned char opad[64]; /*!< HMAC: outer padding */ 43 | } 44 | md5_context; 45 | 46 | #ifdef __cplusplus 47 | extern "C" { 48 | #endif 49 | 50 | /** 51 | * \brief MD5 context setup 52 | * 53 | * \param ctx context to be initialized 54 | */ 55 | void md5_starts( md5_context *ctx ); 56 | 57 | /** 58 | * \brief MD5 process buffer 59 | * 60 | * \param ctx MD5 context 61 | * \param input buffer holding the data 62 | * \param ilen length of the input data 63 | */ 64 | void md5_update( md5_context *ctx, const unsigned char *input, size_t ilen ); 65 | 66 | /** 67 | * \brief MD5 final digest 68 | * 69 | * \param ctx MD5 context 70 | * \param output MD5 checksum result 71 | */ 72 | void md5_finish( md5_context *ctx, unsigned char output[16] ); 73 | 74 | /** 75 | * \brief Output = MD5( input buffer ) 76 | * 77 | * \param input buffer holding the data 78 | * \param ilen length of the input data 79 | * \param output MD5 checksum result 80 | */ 81 | void md5( const unsigned char *input, size_t ilen, unsigned char output[16] ); 82 | 83 | /** 84 | * \brief Output = MD5( file contents ) 85 | * 86 | * \param path input file name 87 | * \param output MD5 checksum result 88 | * 89 | * \return 0 if successful, 1 if fopen failed, 90 | * or 2 if fread failed 91 | */ 92 | int md5_file( const char *path, unsigned char output[16] ); 93 | 94 | /** 95 | * \brief MD5 HMAC context setup 96 | * 97 | * \param ctx HMAC context to be initialized 98 | * \param key HMAC secret key 99 | * \param keylen length of the HMAC key 100 | */ 101 | void md5_hmac_starts( md5_context *ctx, 102 | const unsigned char *key, size_t keylen ); 103 | 104 | /** 105 | * \brief MD5 HMAC process buffer 106 | * 107 | * \param ctx HMAC context 108 | * \param input buffer holding the data 109 | * \param ilen length of the input data 110 | */ 111 | void md5_hmac_update( md5_context *ctx, 112 | const unsigned char *input, size_t ilen ); 113 | 114 | /** 115 | * \brief MD5 HMAC final digest 116 | * 117 | * \param ctx HMAC context 118 | * \param output MD5 HMAC checksum result 119 | */ 120 | void md5_hmac_finish( md5_context *ctx, unsigned char output[16] ); 121 | 122 | /** 123 | * \brief MD5 HMAC context reset 124 | * 125 | * \param ctx HMAC context to be reset 126 | */ 127 | void md5_hmac_reset( md5_context *ctx ); 128 | 129 | /** 130 | * \brief Output = HMAC-MD5( hmac key, input buffer ) 131 | * 132 | * \param key HMAC secret key 133 | * \param keylen length of the HMAC key 134 | * \param input buffer holding the data 135 | * \param ilen length of the input data 136 | * \param output HMAC-MD5 result 137 | */ 138 | void md5_hmac( const unsigned char *key, size_t keylen, 139 | const unsigned char *input, size_t ilen, 140 | unsigned char output[16] ); 141 | 142 | /** 143 | * \brief Checkup routine 144 | * 145 | * \return 0 if successful, or 1 if the test failed 146 | */ 147 | int md5_self_test( int verbose ); 148 | 149 | #ifdef __cplusplus 150 | } 151 | #endif 152 | 153 | #endif /* md5.h */ 154 | -------------------------------------------------------------------------------- /src/hash/xor_hash.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * xor_hash.cpp 3 | * 4 | * Created on: 2011-5-9 5 | * Author: auxten 6 | **/ 7 | #include 8 | #include "xor_hash.h" 9 | #include "../gingko.h" 10 | #include "../log.h" 11 | #include "../job_state.h" 12 | #include "../limit.h" 13 | 14 | /** 15 | * @brief xor hash a given length buf 16 | * 17 | * @see 18 | * @note 19 | * if hval is not 0, use it as the init hash value 20 | * @author auxten 21 | * @date 2011-8-1 22 | **/ 23 | unsigned xor_hash(const void *key, int len, unsigned hval) 24 | { 25 | #if defined(ROT_XOR_HASH) 26 | u_char *p = (u_char *) key; 27 | hval = hval ? hval : 2166136261; 28 | #if defined(HASH_BYTE_NUM_ONCE) 29 | for (int i = 0; i <= len - HASH_BYTE_NUM_ONCE; i += HASH_BYTE_NUM_ONCE) 30 | { 31 | hval = ROLL(hval) ^ p[i]; 32 | hval = ROLL(hval) ^ p[i + 1]; 33 | hval = ROLL(hval) ^ p[i + 2]; 34 | hval = ROLL(hval) ^ p[i + 3]; 35 | #if HASH_BYTE_NUM_ONCE == 8 36 | hval = ROLL(hval) ^ p[i + 4]; 37 | hval = ROLL(hval) ^ p[i + 5]; 38 | hval = ROLL(hval) ^ p[i + 6]; 39 | hval = ROLL(hval) ^ p[i + 7]; 40 | #endif /** HASH_BYTE_NUM_ONCE == 8 **/ 41 | } 42 | /** 43 | * hash the remained bytes 44 | **/ 45 | for (int i = len - len % HASH_BYTE_NUM_ONCE; i < len; i++) 46 | { 47 | hval = ROLL(hval) ^ p[i]; 48 | } 49 | #else 50 | for (int i = 0; i < len; i++) 51 | { 52 | hval = ROLL(hval) ^ p[i]; 53 | } 54 | #endif 55 | 56 | #elif defined(FNV_XOR_HASH) 57 | u_char *p = (u_char *) key; 58 | hval = hval ? hval : 2166136261; 59 | 60 | for (int i = 0; i < len; i++) 61 | { 62 | #if defined(NO_SO_CALLED_FNV_OPTIMIZE) 63 | hval = (hval * 16777619) ^ p[i]; 64 | #else 65 | hval += (hval << 1) + (hval << 4) + (hval << 7) + (hval << 8) + (hval 66 | << 24); 67 | hval ^= p[i]; 68 | #endif 69 | } 70 | 71 | return hval; 72 | #endif /** ROT_XOR_HASH **/ 73 | return hval; 74 | } 75 | 76 | /** 77 | * @brief check if the fnv check sum is OK 78 | * 79 | * @see 80 | * @note 81 | * @author auxten 82 | * @date 2011-8-1 83 | **/ 84 | char digest_ok(void * buf, s_block_t * b) 85 | { 86 | return (xor_hash(buf, b->size, 0) == b->digest); 87 | } 88 | 89 | /** 90 | * @brief xor hash specified block 91 | * 92 | * @see 93 | * @note 94 | * @author auxten 95 | * @date 2011-8-1 96 | **/ 97 | unsigned xor_hash_block(s_job_t * jo, GKO_INT64 block_id, u_char * buf) 98 | { 99 | s_file_t * files = jo->files; 100 | s_block_t * blocks = jo->blocks; 101 | GKO_INT64 read_counter = 0; 102 | GKO_INT64 file_i = (blocks + block_id)->start_f; 103 | GKO_INT64 offset = 0; 104 | int fd; 105 | unsigned tmp_hash = 0; 106 | 107 | if (FAIL_CHECK(-1 == (fd = open((files + ((blocks + block_id)->start_f))->name, 108 | O_RDONLY | O_NOFOLLOW)))) 109 | { 110 | gko_log(WARNING, "file open() error!"); 111 | } 112 | memset(buf, 0, BLOCK_SIZE); 113 | offset = (blocks + block_id)->start_off; 114 | while (read_counter < (blocks + block_id)->size) 115 | { 116 | GKO_INT64 tmp = pread(fd, buf + read_counter, 117 | (blocks + block_id)->size - read_counter, offset); 118 | if (FAIL_CHECK(tmp < 0)) 119 | { 120 | gko_log(WARNING, "pread failed"); 121 | } 122 | if (LIKELY(tmp)) 123 | { 124 | ///printf("read: %ld\n", tmp); 125 | tmp_hash = xor_hash(buf + read_counter, (int) tmp, tmp_hash); 126 | read_counter += tmp; 127 | offset += tmp; 128 | } 129 | else 130 | { 131 | close(fd); 132 | ///if the next if a nonfile then next 133 | file_i = next_f(jo, file_i); 134 | if (FAIL_CHECK(-1 135 | == (fd = open( 136 | (files + ((blocks + block_id)->start_f) 137 | + file_i)->name, O_RDONLY | O_NOFOLLOW)))) 138 | { 139 | fprintf(stderr, "filename: %s\n", 140 | (files + ((blocks + block_id)->start_f) + file_i)->name); 141 | gko_log(WARNING, "filename: %s", 142 | (files + ((blocks + block_id)->start_f) + file_i)->name); 143 | } 144 | offset = 0; 145 | 146 | } 147 | } 148 | (blocks + block_id)->digest = tmp_hash; 149 | /// printf("buf: %d\n", sizeof(buf)); 150 | /// memset(buf, 0, sizeof(buf)); 151 | /// printf("buf: %d\n", sizeof(buf)); 152 | close(fd); 153 | return tmp_hash; 154 | } 155 | 156 | /** 157 | * @brief xor hash the file given 158 | * 159 | * @see 160 | * @note 161 | * @author auxten 162 | * @date 2011-8-1 163 | **/ 164 | unsigned xor_hash_file(unsigned value, FILE * fd, off_t * off, size_t * count, 165 | u_char * buf) 166 | { 167 | fseeko(fd, *off, SEEK_SET); 168 | if (FAIL_CHECK(*count != fread(buf, sizeof(char), *count, fd))) 169 | { 170 | gko_log(FATAL, "fread error"); 171 | } 172 | ///fprintf(stderr, "#######################buf: %s\n", buf); 173 | return xor_hash(buf, *count, value); 174 | } 175 | 176 | /** 177 | * @brief xor hash pthread worker 178 | * 179 | * @see 180 | * @note 181 | * @author auxten 182 | * @date 2011-8-1 183 | **/ 184 | void * xor_hash_worker_f(void * a) 185 | { 186 | /** enable the cancel and make the thread canceled immediately **/ 187 | pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); 188 | pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL); 189 | 190 | hash_worker_thread_arg * arg = (hash_worker_thread_arg *) a; 191 | int t_idx = arg->index; 192 | s_job_t * jo = arg->p; 193 | GKO_INT64 start = arg->range[0]; 194 | GKO_INT64 num = arg->range[1] - arg->range[0]; 195 | gko_log(NOTICE, "range: %lld %lld", arg->range[0], arg->range[1]); 196 | u_char * buf = (jo->hash_buf)[t_idx]; 197 | if (num == 0) 198 | { 199 | delete [] buf; 200 | (jo->hash_buf)[t_idx] = NULL; 201 | pthread_exit((void *) 0); 202 | } 203 | /** 204 | * if include the last block, the total size 205 | * may be smaller than BLOCK_SIZE * num 206 | **/ 207 | GKO_INT64 this_time_sent = 0; 208 | GKO_INT64 total = BLOCK_SIZE * (num - 1) 209 | + (start + num >= jo->block_count ? (jo->blocks + jo->block_count 210 | - 1)->size : BLOCK_SIZE); 211 | GKO_INT64 b = start; 212 | GKO_INT64 f = (jo->blocks + start)->start_f; 213 | size_t block_left = (jo->blocks + start)->size; 214 | GKO_INT64 file_size; 215 | FILE * fd = (FILE *) -1; 216 | off_t offset = (jo->blocks + start)->start_off; 217 | size_t file_left; 218 | 219 | while (total > 0) 220 | { 221 | if (fd == (FILE *) -1) 222 | { 223 | fd = fopen((jo->files + f)->name, "r"); 224 | } 225 | if (FAIL_CHECK(fd <= NULL)) 226 | { 227 | gko_log(WARNING, "xor_hash_worker_f open error"); 228 | jo->job_state = JOB_FILE_OPEN_ERR; 229 | delete [] buf; 230 | (jo->hash_buf)[t_idx] = NULL; 231 | pthread_exit((void *) -2); 232 | } 233 | file_size = (jo->files + f)->size; 234 | file_left = file_size - offset; 235 | 236 | /** read'n'hash the file **/ 237 | if (LIKELY(block_left < file_left)) 238 | { 239 | /**block_left < file_left**/ 240 | (jo->blocks + b)->digest = xor_hash_file((jo->blocks + b)->digest, 241 | fd, &offset, &block_left, buf); 242 | b = next_b(jo, b); 243 | total -= block_left; 244 | (jo->hash_progress)[t_idx] += block_left; 245 | offset += block_left; 246 | this_time_sent = block_left; 247 | block_left = (jo->blocks + b)->size; 248 | } 249 | else 250 | { 251 | (jo->blocks + b)->digest = xor_hash_file((jo->blocks + b)->digest, 252 | fd, &offset, &file_left, buf); 253 | fclose(fd); 254 | fd = (FILE *) -1; 255 | offset = 0; 256 | total -= file_left; 257 | (jo->hash_progress)[t_idx] += file_left; 258 | this_time_sent = file_left; 259 | f = next_f(jo, f); 260 | if (LIKELY(block_left > file_left)) 261 | { 262 | /**block_left > file_left**/ 263 | block_left -= file_left; 264 | } 265 | else 266 | { 267 | /**block_left == file_left**/ 268 | b = next_b(jo, b); 269 | block_left = (jo->blocks + b)->size; 270 | } 271 | } 272 | //mk_seed_limit(this_time_sent, gko.opt.limit_mk_seed_rate); 273 | /// fprintf(stderr, "p_porgress %lld\n", *(arg->p_porgress)); 274 | } 275 | 276 | delete [] buf; 277 | (jo->hash_buf)[t_idx] = NULL; 278 | if (fd != (FILE *) -1) 279 | { 280 | fclose(fd); 281 | fd = (FILE *) -1; 282 | } 283 | gko_log(TRACE, "xor_hash_worker_f returned successfully"); 284 | pthread_exit((void *) 0); 285 | } 286 | // 287 | //void * xor_hash_worker(void * a) 288 | //{ 289 | // hash_worker_thread_arg * arg = (hash_worker_thread_arg *) a; 290 | // for (GKO_INT64 i = arg->range[0]; i < arg->range[1]; i++) 291 | // { 292 | // xor_hash_block(arg->p, i, arg->buf); 293 | // } 294 | // 295 | // pthread_exit((void *) 0); 296 | //} 297 | 298 | /** 299 | * @brief xor hash all blocks for a job given 300 | * 301 | * @see 302 | * @note 303 | * @author auxten 304 | * @date 2011-8-1 305 | **/ 306 | int xor_hash_all(s_job_t * jo, hash_worker_thread_arg arg[]) 307 | { 308 | pthread_attr_t attr; 309 | ///mutex for up bandwidth limit 310 | 311 | /** 312 | * if size == 13 XOR_HASH_TNUM == 4;range will be 0~3 3~6 6~9 9~13 313 | **/ 314 | for (int i = 0; i < XOR_HASH_TNUM; i++) 315 | { 316 | (jo->hash_buf)[i] = new u_char[BLOCK_SIZE + 1]; 317 | if (FAIL_CHECK(!(jo->hash_buf)[i])) 318 | { 319 | gko_log(FATAL, "new fuv hash buf failed"); 320 | return -1; 321 | } 322 | memset((jo->hash_buf)[i], 0, BLOCK_SIZE + 1); 323 | arg[i].index = i; 324 | arg[i].range[0] = jo->block_count / XOR_HASH_TNUM * i; 325 | arg[i].range[1] = jo->block_count / XOR_HASH_TNUM * (i + 1); 326 | arg[i].p = jo; 327 | (jo->hash_progress)[i] = 0; 328 | } 329 | /** last thread must make the work done **/ 330 | arg[XOR_HASH_TNUM - 1].range[1] = jo->block_count; 331 | 332 | /** initialize the pthread attrib for every worker **/ 333 | pthread_attr_init(&attr); 334 | /** set attr state for join() in the mother **/ 335 | pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); 336 | pthread_attr_setstacksize(&attr, MYSTACKSIZE); 337 | 338 | for (int i = 0; i < XOR_HASH_TNUM; i++) 339 | { 340 | pthread_create(&jo->hash_worker[i], &attr, xor_hash_worker_f, &(arg[i])); 341 | } 342 | /// pthread_attr_* moved to main(); 343 | 344 | return 0; 345 | } 346 | -------------------------------------------------------------------------------- /src/hash/xor_hash.h: -------------------------------------------------------------------------------- 1 | /** 2 | * xor_hash.h 3 | * 4 | * Created on: 2011-5-9 5 | * Author: auxten 6 | **/ 7 | #include "../gingko.h" 8 | 9 | #ifndef XOR_HASH_H_ 10 | #define XOR_HASH_H_ 11 | 12 | /// shift Macro 13 | //#define ROLL(h) (((h) << 7) ^ ((h) >> 25)) 14 | #define ROLL(h) (h * 16777619) 15 | 16 | /// xor hash a given length buf 17 | unsigned xor_hash(const void *key, int len, unsigned hval); 18 | /// check if the fnv check sum is OK 19 | char digest_ok(void * buf, s_block_t * b); 20 | /// xor hash all blocks for a job given 21 | int xor_hash_all(s_job_t * jo, hash_worker_thread_arg arg[]); 22 | #endif /** XOR_HASH_H_ **/ 23 | -------------------------------------------------------------------------------- /src/install-sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # install - install a program, script, or datafile 3 | 4 | scriptversion=2011-11-20.07; # UTC 5 | 6 | # This originates from X11R5 (mit/util/scripts/install.sh), which was 7 | # later released in X11R6 (xc/config/util/install.sh) with the 8 | # following copyright and license. 9 | # 10 | # Copyright (C) 1994 X Consortium 11 | # 12 | # Permission is hereby granted, free of charge, to any person obtaining a copy 13 | # of this software and associated documentation files (the "Software"), to 14 | # deal in the Software without restriction, including without limitation the 15 | # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 16 | # sell copies of the Software, and to permit persons to whom the Software is 17 | # furnished to do so, subject to the following conditions: 18 | # 19 | # The above copyright notice and this permission notice shall be included in 20 | # all copies or substantial portions of the Software. 21 | # 22 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 | # X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN 26 | # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC- 27 | # TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 28 | # 29 | # Except as contained in this notice, the name of the X Consortium shall not 30 | # be used in advertising or otherwise to promote the sale, use or other deal- 31 | # ings in this Software without prior written authorization from the X Consor- 32 | # tium. 33 | # 34 | # 35 | # FSF changes to this file are in the public domain. 36 | # 37 | # Calling this script install-sh is preferred over install.sh, to prevent 38 | # 'make' implicit rules from creating a file called install from it 39 | # when there is no Makefile. 40 | # 41 | # This script is compatible with the BSD install script, but was written 42 | # from scratch. 43 | 44 | nl=' 45 | ' 46 | IFS=" "" $nl" 47 | 48 | # set DOITPROG to echo to test this script 49 | 50 | # Don't use :- since 4.3BSD and earlier shells don't like it. 51 | doit=${DOITPROG-} 52 | if test -z "$doit"; then 53 | doit_exec=exec 54 | else 55 | doit_exec=$doit 56 | fi 57 | 58 | # Put in absolute file names if you don't have them in your path; 59 | # or use environment vars. 60 | 61 | chgrpprog=${CHGRPPROG-chgrp} 62 | chmodprog=${CHMODPROG-chmod} 63 | chownprog=${CHOWNPROG-chown} 64 | cmpprog=${CMPPROG-cmp} 65 | cpprog=${CPPROG-cp} 66 | mkdirprog=${MKDIRPROG-mkdir} 67 | mvprog=${MVPROG-mv} 68 | rmprog=${RMPROG-rm} 69 | stripprog=${STRIPPROG-strip} 70 | 71 | posix_glob='?' 72 | initialize_posix_glob=' 73 | test "$posix_glob" != "?" || { 74 | if (set -f) 2>/dev/null; then 75 | posix_glob= 76 | else 77 | posix_glob=: 78 | fi 79 | } 80 | ' 81 | 82 | posix_mkdir= 83 | 84 | # Desired mode of installed file. 85 | mode=0755 86 | 87 | chgrpcmd= 88 | chmodcmd=$chmodprog 89 | chowncmd= 90 | mvcmd=$mvprog 91 | rmcmd="$rmprog -f" 92 | stripcmd= 93 | 94 | src= 95 | dst= 96 | dir_arg= 97 | dst_arg= 98 | 99 | copy_on_change=false 100 | no_target_directory= 101 | 102 | usage="\ 103 | Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE 104 | or: $0 [OPTION]... SRCFILES... DIRECTORY 105 | or: $0 [OPTION]... -t DIRECTORY SRCFILES... 106 | or: $0 [OPTION]... -d DIRECTORIES... 107 | 108 | In the 1st form, copy SRCFILE to DSTFILE. 109 | In the 2nd and 3rd, copy all SRCFILES to DIRECTORY. 110 | In the 4th, create DIRECTORIES. 111 | 112 | Options: 113 | --help display this help and exit. 114 | --version display version info and exit. 115 | 116 | -c (ignored) 117 | -C install only if different (preserve the last data modification time) 118 | -d create directories instead of installing files. 119 | -g GROUP $chgrpprog installed files to GROUP. 120 | -m MODE $chmodprog installed files to MODE. 121 | -o USER $chownprog installed files to USER. 122 | -s $stripprog installed files. 123 | -t DIRECTORY install into DIRECTORY. 124 | -T report an error if DSTFILE is a directory. 125 | 126 | Environment variables override the default commands: 127 | CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG 128 | RMPROG STRIPPROG 129 | " 130 | 131 | while test $# -ne 0; do 132 | case $1 in 133 | -c) ;; 134 | 135 | -C) copy_on_change=true;; 136 | 137 | -d) dir_arg=true;; 138 | 139 | -g) chgrpcmd="$chgrpprog $2" 140 | shift;; 141 | 142 | --help) echo "$usage"; exit $?;; 143 | 144 | -m) mode=$2 145 | case $mode in 146 | *' '* | *' '* | *' 147 | '* | *'*'* | *'?'* | *'['*) 148 | echo "$0: invalid mode: $mode" >&2 149 | exit 1;; 150 | esac 151 | shift;; 152 | 153 | -o) chowncmd="$chownprog $2" 154 | shift;; 155 | 156 | -s) stripcmd=$stripprog;; 157 | 158 | -t) dst_arg=$2 159 | # Protect names problematic for 'test' and other utilities. 160 | case $dst_arg in 161 | -* | [=!]) dst_arg=./$dst_arg;; 162 | esac 163 | shift;; 164 | 165 | -T) no_target_directory=true;; 166 | 167 | --version) echo "$0 $scriptversion"; exit $?;; 168 | 169 | --) shift 170 | break;; 171 | 172 | -*) echo "$0: invalid option: $1" >&2 173 | exit 1;; 174 | 175 | *) break;; 176 | esac 177 | shift 178 | done 179 | 180 | if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then 181 | # When -d is used, all remaining arguments are directories to create. 182 | # When -t is used, the destination is already specified. 183 | # Otherwise, the last argument is the destination. Remove it from $@. 184 | for arg 185 | do 186 | if test -n "$dst_arg"; then 187 | # $@ is not empty: it contains at least $arg. 188 | set fnord "$@" "$dst_arg" 189 | shift # fnord 190 | fi 191 | shift # arg 192 | dst_arg=$arg 193 | # Protect names problematic for 'test' and other utilities. 194 | case $dst_arg in 195 | -* | [=!]) dst_arg=./$dst_arg;; 196 | esac 197 | done 198 | fi 199 | 200 | if test $# -eq 0; then 201 | if test -z "$dir_arg"; then 202 | echo "$0: no input file specified." >&2 203 | exit 1 204 | fi 205 | # It's OK to call 'install-sh -d' without argument. 206 | # This can happen when creating conditional directories. 207 | exit 0 208 | fi 209 | 210 | if test -z "$dir_arg"; then 211 | do_exit='(exit $ret); exit $ret' 212 | trap "ret=129; $do_exit" 1 213 | trap "ret=130; $do_exit" 2 214 | trap "ret=141; $do_exit" 13 215 | trap "ret=143; $do_exit" 15 216 | 217 | # Set umask so as not to create temps with too-generous modes. 218 | # However, 'strip' requires both read and write access to temps. 219 | case $mode in 220 | # Optimize common cases. 221 | *644) cp_umask=133;; 222 | *755) cp_umask=22;; 223 | 224 | *[0-7]) 225 | if test -z "$stripcmd"; then 226 | u_plus_rw= 227 | else 228 | u_plus_rw='% 200' 229 | fi 230 | cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;; 231 | *) 232 | if test -z "$stripcmd"; then 233 | u_plus_rw= 234 | else 235 | u_plus_rw=,u+rw 236 | fi 237 | cp_umask=$mode$u_plus_rw;; 238 | esac 239 | fi 240 | 241 | for src 242 | do 243 | # Protect names problematic for 'test' and other utilities. 244 | case $src in 245 | -* | [=!]) src=./$src;; 246 | esac 247 | 248 | if test -n "$dir_arg"; then 249 | dst=$src 250 | dstdir=$dst 251 | test -d "$dstdir" 252 | dstdir_status=$? 253 | else 254 | 255 | # Waiting for this to be detected by the "$cpprog $src $dsttmp" command 256 | # might cause directories to be created, which would be especially bad 257 | # if $src (and thus $dsttmp) contains '*'. 258 | if test ! -f "$src" && test ! -d "$src"; then 259 | echo "$0: $src does not exist." >&2 260 | exit 1 261 | fi 262 | 263 | if test -z "$dst_arg"; then 264 | echo "$0: no destination specified." >&2 265 | exit 1 266 | fi 267 | dst=$dst_arg 268 | 269 | # If destination is a directory, append the input filename; won't work 270 | # if double slashes aren't ignored. 271 | if test -d "$dst"; then 272 | if test -n "$no_target_directory"; then 273 | echo "$0: $dst_arg: Is a directory" >&2 274 | exit 1 275 | fi 276 | dstdir=$dst 277 | dst=$dstdir/`basename "$src"` 278 | dstdir_status=0 279 | else 280 | # Prefer dirname, but fall back on a substitute if dirname fails. 281 | dstdir=` 282 | (dirname "$dst") 2>/dev/null || 283 | expr X"$dst" : 'X$.*[^/]$//*[^/][^/]*/*$' \| \ 284 | X"$dst" : 'X$//$[^/]' \| \ 285 | X"$dst" : 'X$//$$' \| \ 286 | X"$dst" : 'X$/$' \| . 2>/dev/null || 287 | echo X"$dst" | 288 | sed '/^X$.*[^/]$\/\/*[^/][^/]*\/*$/{ 289 | s//\1/ 290 | q 291 | } 292 | /^X$\/\/$[^/].*/{ 293 | s//\1/ 294 | q 295 | } 296 | /^X$\/\/$$/{ 297 | s//\1/ 298 | q 299 | } 300 | /^X$\/$.*/{ 301 | s//\1/ 302 | q 303 | } 304 | s/.*/./; q' 305 | ` 306 | 307 | test -d "$dstdir" 308 | dstdir_status=$? 309 | fi 310 | fi 311 | 312 | obsolete_mkdir_used=false 313 | 314 | if test $dstdir_status != 0; then 315 | case $posix_mkdir in 316 | '') 317 | # Create intermediate dirs using mode 755 as modified by the umask. 318 | # This is like FreeBSD 'install' as of 1997-10-28. 319 | umask=`umask` 320 | case $stripcmd.$umask in 321 | # Optimize common cases. 322 | *[2367][2367]) mkdir_umask=$umask;; 323 | .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;; 324 | 325 | *[0-7]) 326 | mkdir_umask=`expr $umask + 22 \ 327 | - $umask % 100 % 40 + $umask % 20 \ 328 | - $umask % 10 % 4 + $umask % 2 329 | `;; 330 | *) mkdir_umask=$umask,go-w;; 331 | esac 332 | 333 | # With -d, create the new directory with the user-specified mode. 334 | # Otherwise, rely on $mkdir_umask. 335 | if test -n "$dir_arg"; then 336 | mkdir_mode=-m$mode 337 | else 338 | mkdir_mode= 339 | fi 340 | 341 | posix_mkdir=false 342 | case $umask in 343 | *[123567][0-7][0-7]) 344 | # POSIX mkdir -p sets u+wx bits regardless of umask, which 345 | # is incompatible with FreeBSD 'install' when (umask & 300) != 0. 346 | ;; 347 | *) 348 | tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$ 349 | trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0 350 | 351 | if (umask $mkdir_umask && 352 | exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1 353 | then 354 | if test -z "$dir_arg" || { 355 | # Check for POSIX incompatibilities with -m. 356 | # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or 357 | # other-writable bit of parent directory when it shouldn't. 358 | # FreeBSD 6.1 mkdir -m -p sets mode of existing directory. 359 | ls_ld_tmpdir=`ls -ld "$tmpdir"` 360 | case $ls_ld_tmpdir in 361 | d????-?r-*) different_mode=700;; 362 | d????-?--*) different_mode=755;; 363 | *) false;; 364 | esac && 365 | $mkdirprog -m$different_mode -p -- "$tmpdir" && { 366 | ls_ld_tmpdir_1=`ls -ld "$tmpdir"` 367 | test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1" 368 | } 369 | } 370 | then posix_mkdir=: 371 | fi 372 | rmdir "$tmpdir/d" "$tmpdir" 373 | else 374 | # Remove any dirs left behind by ancient mkdir implementations. 375 | rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null 376 | fi 377 | trap '' 0;; 378 | esac;; 379 | esac 380 | 381 | if 382 | $posix_mkdir && ( 383 | umask $mkdir_umask && 384 | $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir" 385 | ) 386 | then : 387 | else 388 | 389 | # The umask is ridiculous, or mkdir does not conform to POSIX, 390 | # or it failed possibly due to a race condition. Create the 391 | # directory the slow way, step by step, checking for races as we go. 392 | 393 | case $dstdir in 394 | /*) prefix='/';; 395 | [-=!]*) prefix='./';; 396 | *) prefix='';; 397 | esac 398 | 399 | eval "$initialize_posix_glob" 400 | 401 | oIFS=$IFS 402 | IFS=/ 403 | $posix_glob set -f 404 | set fnord $dstdir 405 | shift 406 | $posix_glob set +f 407 | IFS=$oIFS 408 | 409 | prefixes= 410 | 411 | for d 412 | do 413 | test X"$d" = X && continue 414 | 415 | prefix=$prefix$d 416 | if test -d "$prefix"; then 417 | prefixes= 418 | else 419 | if $posix_mkdir; then 420 | (umask=$mkdir_umask && 421 | $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break 422 | # Don't fail if two instances are running concurrently. 423 | test -d "$prefix" || exit 1 424 | else 425 | case $prefix in 426 | *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;; 427 | *) qprefix=$prefix;; 428 | esac 429 | prefixes="$prefixes '$qprefix'" 430 | fi 431 | fi 432 | prefix=$prefix/ 433 | done 434 | 435 | if test -n "$prefixes"; then 436 | # Don't fail if two instances are running concurrently. 437 | (umask $mkdir_umask && 438 | eval "\$doit_exec \$mkdirprog $prefixes") || 439 | test -d "$dstdir" || exit 1 440 | obsolete_mkdir_used=true 441 | fi 442 | fi 443 | fi 444 | 445 | if test -n "$dir_arg"; then 446 | { test -z "$chowncmd" || $doit $chowncmd "$dst"; } && 447 | { test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } && 448 | { test "$obsolete_mkdir_used$chowncmd$chgrpcmd" = false || 449 | test -z "$chmodcmd" || $doit $chmodcmd $mode "$dst"; } || exit 1 450 | else 451 | 452 | # Make a couple of temp file names in the proper directory. 453 | dsttmp=$dstdir/_inst.$$_ 454 | rmtmp=$dstdir/_rm.$$_ 455 | 456 | # Trap to clean up those temp files at exit. 457 | trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0 458 | 459 | # Copy the file name to the temp name. 460 | (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") && 461 | 462 | # and set any options; do chmod last to preserve setuid bits. 463 | # 464 | # If any of these fail, we abort the whole thing. If we want to 465 | # ignore errors from any of these, just make sure not to ignore 466 | # errors from the above "$doit $cpprog $src $dsttmp" command. 467 | # 468 | { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } && 469 | { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } && 470 | { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } && 471 | { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } && 472 | 473 | # If -C, don't bother to copy if it wouldn't change the file. 474 | if $copy_on_change && 475 | old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` && 476 | new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` && 477 | 478 | eval "$initialize_posix_glob" && 479 | $posix_glob set -f && 480 | set X $old && old=:$2:$4:$5:$6 && 481 | set X $new && new=:$2:$4:$5:$6 && 482 | $posix_glob set +f && 483 | 484 | test "$old" = "$new" && 485 | $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1 486 | then 487 | rm -f "$dsttmp" 488 | else 489 | # Rename the file to the real destination. 490 | $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null || 491 | 492 | # The rename failed, perhaps because mv can't rename something else 493 | # to itself, or perhaps because mv is so ancient that it does not 494 | # support -f. 495 | { 496 | # Now remove or move aside any old file at destination location. 497 | # We try this two ways since rm can't unlink itself on some 498 | # systems and the destination file might be busy for other 499 | # reasons. In this case, the final cleanup might fail but the new 500 | # file should still install successfully. 501 | { 502 | test ! -f "$dst" || 503 | $doit $rmcmd -f "$dst" 2>/dev/null || 504 | { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null && 505 | { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; } 506 | } || 507 | { echo "$0: cannot unlink or rename $dst" >&2 508 | (exit 1); exit 1 509 | } 510 | } && 511 | 512 | # Now rename the file to the real destination. 513 | $doit $mvcmd "$dsttmp" "$dst" 514 | } 515 | fi || exit 1 516 | 517 | trap '' 0 518 | fi 519 | done 520 | 521 | # Local variables: 522 | # eval: (add-hook 'write-file-hooks 'time-stamp) 523 | # time-stamp-start: "scriptversion=" 524 | # time-stamp-format: "%:y-%02m-%02d.%02H" 525 | # time-stamp-time-zone: "UTC" 526 | # time-stamp-end: "; # UTC" 527 | # End: 528 | -------------------------------------------------------------------------------- /src/job_state.h: -------------------------------------------------------------------------------- 1 | /** 2 | * job_state.h 3 | * 4 | * Created on: 2011-7-15 5 | * Author: auxten 6 | **/ 7 | 8 | #ifndef JOB_STATE_H_ 9 | #define JOB_STATE_H_ 10 | 11 | /// job init succeed 12 | static const unsigned int JOB_SUCC = 101; 13 | /// job init failed due to unsupported file type 14 | static const unsigned int JOB_FILE_TYPE_ERR = 102; 15 | /// job init failed due to can't open the file 16 | static const unsigned int JOB_FILE_OPEN_ERR = 103; 17 | /// job init failed due to recurse error 18 | static const unsigned int JOB_RECURSE_ERR = 104; 19 | /// job is going to be erased 20 | static const unsigned int JOB_TO_BE_ERASED = 105; 21 | /// job is joined, ie: g_job is ready 22 | static const unsigned int JOB_IS_JOINED = 106; 23 | /// job init failed due to recurse error 24 | static const unsigned int JOB_RECURSE_DONE = 107; 25 | 26 | 27 | #endif /** JOB_STATE_H_ **/ 28 | -------------------------------------------------------------------------------- /src/limit.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * limit.cpp 3 | * 4 | * Created on: 2011-8-4 5 | * Author: auxten 6 | **/ 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "gingko.h" 17 | 18 | 19 | ///mutex for up bandwidth limit 20 | static pthread_mutex_t g_bw_up_mutex = PTHREAD_MUTEX_INITIALIZER; 21 | ///mutex for down bandwidth limit 22 | static pthread_mutex_t g_bw_down_mutex = PTHREAD_MUTEX_INITIALIZER; 23 | ///mutex for up bandwidth limit 24 | static pthread_mutex_t g_disk_r_mutex = PTHREAD_MUTEX_INITIALIZER; 25 | ///mutex for down bandwidth limit 26 | static pthread_mutex_t g_disk_w_mutex = PTHREAD_MUTEX_INITIALIZER; 27 | ///mutex for make seed limit 28 | static pthread_mutex_t g_mk_seed_mutex = PTHREAD_MUTEX_INITIALIZER; 29 | 30 | /** 31 | * @brief limit down rate 32 | * 33 | * @see 34 | * @note bwlimit modified from scp 35 | * @author auxten 36 | * @date 2011-8-1 37 | **/ 38 | void bw_down_limit(int amount, int limit_rate) 39 | { 40 | static struct timeval bw_down_start; 41 | static struct timeval bw_down_end; 42 | static int down_lamt; 43 | static int down_thresh = 16384; 44 | GKO_UINT64 waitlen; 45 | struct timespec ts; 46 | struct timespec rm; 47 | 48 | if (amount <= 0 || limit_rate <= 0) 49 | { 50 | return; 51 | } 52 | pthread_mutex_lock(&g_bw_down_mutex); 53 | 54 | if (UNLIKELY(!timerisset(&bw_down_start))) 55 | { 56 | gettimeofday(&bw_down_start, NULL); 57 | goto DISK_W_UNLOCK_RET; 58 | } 59 | 60 | down_lamt += amount; 61 | if (down_lamt < down_thresh) 62 | { 63 | goto DISK_W_UNLOCK_RET; 64 | } 65 | 66 | gettimeofday(&bw_down_end, NULL); 67 | timersub(&bw_down_end, &bw_down_start, &bw_down_end); 68 | if (!timerisset(&bw_down_end)) 69 | { 70 | goto DISK_W_UNLOCK_RET; 71 | } 72 | 73 | waitlen = (GKO_UINT64) 1000000L * down_lamt / limit_rate; 74 | 75 | bw_down_start.tv_sec = waitlen / 1000000L; 76 | bw_down_start.tv_usec = waitlen % 1000000L; 77 | 78 | if (timercmp(&bw_down_start, &bw_down_end, >)) 79 | { 80 | timersub(&bw_down_start, &bw_down_end, &bw_down_end); 81 | 82 | /** Adjust the wait time **/ 83 | if (bw_down_end.tv_sec) 84 | { 85 | down_thresh /= 2; 86 | if (down_thresh < 2048) 87 | { 88 | down_thresh = 2048; 89 | } 90 | } 91 | else if (bw_down_end.tv_usec < 100) 92 | { 93 | down_thresh *= 2; 94 | if (down_thresh > 32768) 95 | { 96 | down_thresh = 32768; 97 | } 98 | } 99 | 100 | TIMEVAL_TO_TIMESPEC(&bw_down_end, &ts); 101 | ///gko_log(WARNING, "sleep for: %d usec", (&bw_down_end)->tv_usec); 102 | while (nanosleep(&ts, &rm) == -1) 103 | { 104 | if (errno != EINTR) 105 | { 106 | break; 107 | } 108 | ts = rm; 109 | } 110 | } 111 | 112 | down_lamt = 0; 113 | gettimeofday(&bw_down_start, NULL); 114 | 115 | DISK_W_UNLOCK_RET: 116 | pthread_mutex_unlock(&g_bw_down_mutex); 117 | return; 118 | } 119 | 120 | /** 121 | * @brief limit up rate 122 | * 123 | * @see 124 | * @note bwlimit modified from scp 125 | * @author auxten 126 | * @date 2011-8-1 127 | **/ 128 | void bw_up_limit(int amount, int limit_rate) 129 | { 130 | static struct timeval bw_up_start; 131 | static struct timeval bw_up_end; 132 | static int up_lamt; 133 | static int up_thresh = 16384; 134 | GKO_UINT64 waitlen; 135 | struct timespec ts; 136 | struct timespec rm; 137 | 138 | if (amount <= 0 || limit_rate <= 0) 139 | { 140 | return; 141 | } 142 | pthread_mutex_lock(&g_bw_up_mutex); 143 | 144 | if (UNLIKELY(!timerisset(&bw_up_start))) 145 | { 146 | gettimeofday(&bw_up_start, NULL); 147 | goto UP_UNLOCK_RET; 148 | } 149 | 150 | up_lamt += amount; 151 | if (up_lamt < up_thresh) 152 | { 153 | goto UP_UNLOCK_RET; 154 | } 155 | 156 | gettimeofday(&bw_up_end, NULL); 157 | timersub(&bw_up_end, &bw_up_start, &bw_up_end); 158 | if (!timerisset(&bw_up_end)) 159 | { 160 | goto UP_UNLOCK_RET; 161 | } 162 | 163 | waitlen = (GKO_UINT64) 1000000L * up_lamt / limit_rate; 164 | 165 | bw_up_start.tv_sec = waitlen / 1000000L; 166 | bw_up_start.tv_usec = waitlen % 1000000L; 167 | 168 | if (timercmp(&bw_up_start, &bw_up_end, >)) 169 | { 170 | timersub(&bw_up_start, &bw_up_end, &bw_up_end); 171 | 172 | /** Adjust the wait time **/ 173 | if (bw_up_end.tv_sec) 174 | { 175 | up_thresh /= 2; 176 | if (up_thresh < 2048) 177 | { 178 | up_thresh = 2048; 179 | } 180 | } 181 | else if (bw_up_end.tv_usec < 100) 182 | { 183 | up_thresh *= 2; 184 | if (up_thresh > 32768) 185 | { 186 | up_thresh = 32768; 187 | } 188 | } 189 | 190 | TIMEVAL_TO_TIMESPEC(&bw_up_end, &ts); 191 | ///gko_log(WARNING, "sleep for: %d usec", (&bw_up_end)->tv_usec); 192 | while (nanosleep(&ts, &rm) == -1) 193 | { 194 | if (errno != EINTR) 195 | { 196 | break; 197 | } 198 | ts = rm; 199 | } 200 | } 201 | 202 | up_lamt = 0; 203 | gettimeofday(&bw_up_start, NULL); 204 | 205 | UP_UNLOCK_RET: 206 | pthread_mutex_unlock(&g_bw_up_mutex); 207 | return; 208 | } 209 | 210 | /** 211 | * @brief limit disk write rate 212 | * 213 | * @see 214 | * @note bwlimit modified from scp 215 | * @author auxten 216 | * @date 2011-8-1 217 | **/ 218 | void disk_w_limit(int amount, int limit_rate) 219 | { 220 | static struct timeval disk_w_start; 221 | static struct timeval disk_w_end; 222 | static int write_lamt; 223 | static int write_thresh = 16384; 224 | GKO_UINT64 waitlen; 225 | struct timespec ts; 226 | struct timespec rm; 227 | 228 | if (amount <= 0 || limit_rate <= 0) 229 | { 230 | return; 231 | } 232 | pthread_mutex_lock(&g_disk_w_mutex); 233 | 234 | if (UNLIKELY(!timerisset(&disk_w_start))) 235 | { 236 | gettimeofday(&disk_w_start, NULL); 237 | goto DISK_W_UNLOCK_RET; 238 | } 239 | 240 | write_lamt += amount; 241 | if (write_lamt < write_thresh) 242 | { 243 | goto DISK_W_UNLOCK_RET; 244 | } 245 | 246 | gettimeofday(&disk_w_end, NULL); 247 | timersub(&disk_w_end, &disk_w_start, &disk_w_end); 248 | if (!timerisset(&disk_w_end)) 249 | { 250 | goto DISK_W_UNLOCK_RET; 251 | } 252 | 253 | waitlen = (GKO_UINT64) 1000000L * write_lamt / limit_rate; 254 | 255 | disk_w_start.tv_sec = waitlen / 1000000L; 256 | disk_w_start.tv_usec = waitlen % 1000000L; 257 | 258 | if (timercmp(&disk_w_start, &disk_w_end, >)) 259 | { 260 | timersub(&disk_w_start, &disk_w_end, &disk_w_end); 261 | 262 | /** Adjust the wait time **/ 263 | if (disk_w_end.tv_sec) 264 | { 265 | write_thresh /= 2; 266 | if (write_thresh < 2048) 267 | { 268 | write_thresh = 2048; 269 | } 270 | } 271 | else if (disk_w_end.tv_usec < 100) 272 | { 273 | write_thresh *= 2; 274 | if (write_thresh > 32768) 275 | { 276 | write_thresh = 32768; 277 | } 278 | } 279 | 280 | TIMEVAL_TO_TIMESPEC(&disk_w_end, &ts); 281 | ///gko_log(WARNING, "sleep for: %d usec", (&disk_w_end)->tv_usec); 282 | while (nanosleep(&ts, &rm) == -1) 283 | { 284 | if (errno != EINTR) 285 | { 286 | break; 287 | } 288 | ts = rm; 289 | } 290 | } 291 | 292 | write_lamt = 0; 293 | gettimeofday(&disk_w_start, NULL); 294 | 295 | DISK_W_UNLOCK_RET: 296 | pthread_mutex_unlock(&g_disk_w_mutex); 297 | return; 298 | } 299 | 300 | /** 301 | * @brief limit disk read rate 302 | * 303 | * @see 304 | * @note bwlimit modified from scp 305 | * @author auxten 306 | * @date 2011-8-1 307 | **/ 308 | void disk_r_limit(int amount, int limit_rate) 309 | { 310 | static struct timeval disk_r_start; 311 | static struct timeval disk_r_end; 312 | static int read_lamt; 313 | static int read_thresh = 16384; 314 | GKO_UINT64 waitlen; 315 | struct timespec ts; 316 | struct timespec rm; 317 | 318 | if (amount <= 0 || limit_rate <= 0) 319 | { 320 | return; 321 | } 322 | pthread_mutex_lock(&g_disk_r_mutex); 323 | 324 | if (UNLIKELY(!timerisset(&disk_r_start))) 325 | { 326 | gettimeofday(&disk_r_start, NULL); 327 | goto DISK_R_UNLOCK_RET; 328 | } 329 | 330 | read_lamt += amount; 331 | if (read_lamt < read_thresh) 332 | { 333 | goto DISK_R_UNLOCK_RET; 334 | } 335 | 336 | gettimeofday(&disk_r_end, NULL); 337 | timersub(&disk_r_end, &disk_r_start, &disk_r_end); 338 | if (!timerisset(&disk_r_end)) 339 | { 340 | goto DISK_R_UNLOCK_RET; 341 | } 342 | 343 | waitlen = (GKO_UINT64) 1000000L * read_lamt / limit_rate; 344 | 345 | disk_r_start.tv_sec = waitlen / 1000000L; 346 | disk_r_start.tv_usec = waitlen % 1000000L; 347 | 348 | if (timercmp(&disk_r_start, &disk_r_end, >)) 349 | { 350 | timersub(&disk_r_start, &disk_r_end, &disk_r_end); 351 | 352 | /** Adjust the wait time **/ 353 | if (disk_r_end.tv_sec) 354 | { 355 | read_thresh /= 2; 356 | if (read_thresh < 2048) 357 | { 358 | read_thresh = 2048; 359 | } 360 | } 361 | else if (disk_r_end.tv_usec < 100) 362 | { 363 | read_thresh *= 2; 364 | if (read_thresh > 32768) 365 | { 366 | read_thresh = 32768; 367 | } 368 | } 369 | 370 | TIMEVAL_TO_TIMESPEC(&disk_r_end, &ts); 371 | ///gko_log(WARNING, "sleep for: %d usec", (&disk_r_end)->tv_usec); 372 | while (nanosleep(&ts, &rm) == -1) 373 | { 374 | if (errno != EINTR) 375 | { 376 | break; 377 | } 378 | ts = rm; 379 | } 380 | } 381 | 382 | read_lamt = 0; 383 | gettimeofday(&disk_r_start, NULL); 384 | 385 | DISK_R_UNLOCK_RET: 386 | pthread_mutex_unlock(&g_disk_r_mutex); 387 | return; 388 | } 389 | 390 | 391 | /** 392 | * @brief limit make seed rate 393 | * 394 | * @see 395 | * @note bwlimit modified from scp 396 | * @author auxten 397 | * @date 2011-8-1 398 | **/ 399 | void mk_seed_limit(int amount, int limit_rate) 400 | { 401 | static struct timeval mk_seed_start; 402 | static struct timeval mk_seed_end; 403 | static int mk_seed_lamt; 404 | static int mk_seed_thresh = 16384; 405 | GKO_UINT64 waitlen; 406 | struct timespec ts; 407 | struct timespec rm; 408 | if (amount <= 0 || limit_rate <= 0) 409 | { 410 | return; 411 | } 412 | pthread_mutex_lock(&g_mk_seed_mutex); 413 | 414 | if (UNLIKELY(!timerisset(&mk_seed_start))) 415 | { 416 | gettimeofday(&mk_seed_start, NULL); 417 | goto MK_SEED_UNLOCK_RET; 418 | } 419 | 420 | mk_seed_lamt += amount; 421 | if (mk_seed_lamt < mk_seed_thresh) 422 | { 423 | goto MK_SEED_UNLOCK_RET; 424 | } 425 | 426 | gettimeofday(&mk_seed_end, NULL); 427 | timersub(&mk_seed_end, &mk_seed_start, &mk_seed_end); 428 | if (!timerisset(&mk_seed_end)) 429 | { 430 | goto MK_SEED_UNLOCK_RET; 431 | } 432 | 433 | waitlen = (GKO_UINT64) 1000000L * mk_seed_lamt / limit_rate; 434 | 435 | mk_seed_start.tv_sec = waitlen / 1000000L; 436 | mk_seed_start.tv_usec = waitlen % 1000000L; 437 | 438 | if (timercmp(&mk_seed_start, &mk_seed_end, >)) 439 | { 440 | timersub(&mk_seed_start, &mk_seed_end, &mk_seed_end); 441 | 442 | /** Adjust the wait time **/ 443 | if (mk_seed_end.tv_sec) 444 | { 445 | mk_seed_thresh /= 2; 446 | if (mk_seed_thresh < 2048) 447 | { 448 | mk_seed_thresh = 2048; 449 | } 450 | } 451 | else if (mk_seed_end.tv_usec < 100) 452 | { 453 | mk_seed_thresh *= 2; 454 | if (mk_seed_thresh > 32768) 455 | { 456 | mk_seed_thresh = 32768; 457 | } 458 | } 459 | 460 | TIMEVAL_TO_TIMESPEC(&mk_seed_end, &ts); 461 | ///gko_log(WARNING, "sleep for: %d usec", (&mk_seed_end)->tv_usec); 462 | while (nanosleep(&ts, &rm) == -1) 463 | { 464 | if (errno != EINTR) 465 | { 466 | break; 467 | } 468 | ts = rm; 469 | } 470 | } 471 | 472 | mk_seed_lamt = 0; 473 | gettimeofday(&mk_seed_start, NULL); 474 | 475 | MK_SEED_UNLOCK_RET: 476 | pthread_mutex_unlock(&g_mk_seed_mutex); 477 | return; 478 | } 479 | 480 | -------------------------------------------------------------------------------- /src/limit.h: -------------------------------------------------------------------------------- 1 | /** 2 | * limit.h 3 | * 4 | * Created on: 2011-8-4 5 | * Author: auxten 6 | **/ 7 | 8 | #ifndef LIMIT_H_ 9 | #define LIMIT_H_ 10 | 11 | /// limit download rate 12 | void bw_down_limit(int amount, int limit_rate); 13 | /// limit upload rate 14 | void bw_up_limit(int amount, int limit_rate); 15 | /// limit disk write rate 16 | void disk_w_limit(int amount, int limit_rate); 17 | /// limit disk read rate 18 | void disk_r_limit(int amount, int limit_rate); 19 | /// limit make seed rate 20 | void mk_seed_limit(int amount, int limit_rate); 21 | 22 | #endif /** LIMIT_H_ **/ 23 | -------------------------------------------------------------------------------- /src/log.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * log.cpp 3 | * 4 | * Created on: 2011-7-13 5 | * Author: auxten 6 | **/ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "gingko.h" 17 | #include "log.h" 18 | 19 | extern s_gingko_global_t gko; 20 | pthread_mutex_t logcut_lock = PTHREAD_MUTEX_INITIALIZER; 21 | 22 | /** 23 | * @brief loglevel 24 | * 25 | * @see 26 | * @note 27 | * @author auxten 28 | * @date 2011-8-1 29 | **/ 30 | static const char * LOG_DIC[] = 31 | { "FATAL: ", "WARNING: ", "NOTICE: ", "TRACE: ", "DEBUG: ", }; 32 | 33 | /** 34 | * @brief generate the time string according to the TIME_FORMAT 35 | * 36 | * @see 37 | * @note 38 | * @author auxten 39 | * @date 2011-8-1 40 | **/ 41 | char * gettimestr(char * time, const char * format) 42 | { 43 | struct timeval tv; 44 | struct tm ltime; 45 | time_t curtime; 46 | gettimeofday(&tv, NULL); 47 | curtime = tv.tv_sec; 48 | ///Format time 49 | strftime(time, 25, format, localtime_r(&curtime, <ime)); 50 | return time; 51 | } 52 | 53 | /** 54 | * @brief log handler 55 | * 56 | * @see 57 | * @note 58 | * @author auxten 59 | * @date 2011-8-1 60 | **/ 61 | void _gko_log(const u_char log_level, const char *fmt, ...) 62 | { 63 | if (gko.opt.to_debug || log_level < DEBUG ) 64 | { 65 | int errnum = errno; 66 | va_list args; 67 | va_start(args, fmt); 68 | char logstr[256] = {'\0'}; 69 | char oldlogpath[MAX_PATH_LEN] = {'\0'}; 70 | static FILE * lastfp = NULL; 71 | static GKO_INT64 counter = 1; 72 | 73 | snprintf(logstr, sizeof(logstr), "%s[%u]", LOG_DIC[log_level], gko_gettid()); 74 | gettimestr(logstr + strlen(logstr), TIME_FORMAT); 75 | vsnprintf(logstr + strlen(logstr), sizeof(logstr) - strlen(logstr), fmt, 76 | args); 77 | if (log_level < NOTICE) 78 | { 79 | snprintf(logstr + strlen(logstr), sizeof(logstr) - strlen(logstr), 80 | "; "); 81 | strerror_r(errnum, logstr + strlen(logstr), 82 | sizeof(logstr) - strlen(logstr)); 83 | } 84 | 85 | pthread_mutex_lock(&logcut_lock); 86 | if (gko.opt.logpath[0] == '\0') 87 | { 88 | gko.log_fp = stdout; 89 | } 90 | else 91 | { 92 | counter ++; 93 | if (counter % MAX_LOG_REOPEN_LINE == 0) 94 | { 95 | if (lastfp) 96 | { 97 | fclose(lastfp); 98 | } 99 | lastfp = gko.log_fp; 100 | if (counter % MAX_LOG_LINE == 0) 101 | { 102 | strncpy(oldlogpath, gko.opt.logpath, MAX_PATH_LEN); 103 | gettimestr(oldlogpath + strlen(oldlogpath), OLD_LOG_TIME); 104 | rename(gko.opt.logpath, oldlogpath); 105 | } 106 | gko.log_fp = fopen(gko.opt.logpath, "a+"); 107 | } 108 | if(UNLIKELY(! gko.log_fp)) 109 | { 110 | gko.log_fp = fopen(gko.opt.logpath, "a+"); 111 | if(! gko.log_fp) 112 | { 113 | perror("Cann't open log file"); 114 | exit(1); 115 | } 116 | } 117 | } 118 | fprintf(gko.log_fp, "%s\n", logstr); 119 | fflush(gko.log_fp); 120 | pthread_mutex_unlock(&logcut_lock); 121 | 122 | va_end(args); 123 | } 124 | return; 125 | } 126 | 127 | int lock_log(void) 128 | { 129 | return pthread_mutex_lock(&logcut_lock); 130 | } 131 | -------------------------------------------------------------------------------- /src/log.h: -------------------------------------------------------------------------------- 1 | /** 2 | * log.h 3 | * 4 | * Created on: 2011-7-13 5 | * Author: auxten 6 | **/ 7 | 8 | #ifndef LOG_H_ 9 | #define LOG_H_ 10 | 11 | ///log level 12 | static const u_char FATAL = 0; 13 | static const u_char WARNING = 1; 14 | static const u_char NOTICE = 2; 15 | static const u_char TRACE = 3; 16 | static const u_char DEBUG = 4; 17 | 18 | /// to print the file line func 19 | /// MUST NOT use it in "const char *fmt, ..." type func 20 | #define FLF(a) "{%s:%d %s} %s",__FILE__,__LINE__,__func__,#a 21 | 22 | /// log handler 23 | void _gko_log(const u_char log_level, const char *fmt, ...); 24 | int lock_log(void); 25 | 26 | /// macro using to print the file and line in log for easy debuging 27 | /// when it is not in releasing build 28 | #ifdef NDEBUG 29 | #define gko_log(log_level, fmt, arg...) \ 30 | do {\ 31 | _gko_log(log_level, fmt, ##arg); \ 32 | } while(0) 33 | #else 34 | #define gko_log(log_level, fmt, arg...) \ 35 | do {\ 36 | _gko_log(log_level, fmt"[%s:%d]", ##arg, __FILE__, __LINE__);\ 37 | } while (0) 38 | #endif 39 | 40 | #endif /** LOG_H_ **/ 41 | -------------------------------------------------------------------------------- /src/missing: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # Common stub for a few missing GNU programs while installing. 3 | 4 | scriptversion=2012-01-06.18; # UTC 5 | 6 | # Copyright (C) 1996-2012 Free Software Foundation, Inc. 7 | # Originally by Fran,cois Pinard , 1996. 8 | 9 | # This program is free software; you can redistribute it and/or modify 10 | # it under the terms of the GNU General Public License as published by 11 | # the Free Software Foundation; either version 2, or (at your option) 12 | # any later version. 13 | 14 | # This program is distributed in the hope that it will be useful, 15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 | # GNU General Public License for more details. 18 | 19 | # You should have received a copy of the GNU General Public License 20 | # along with this program. If not, see . 21 | 22 | # As a special exception to the GNU General Public License, if you 23 | # distribute this file as part of a program that contains a 24 | # configuration script generated by Autoconf, you may include it under 25 | # the same distribution terms that you use for the rest of that program. 26 | 27 | if test $# -eq 0; then 28 | echo 1>&2 "Try '$0 --help' for more information" 29 | exit 1 30 | fi 31 | 32 | run=: 33 | sed_output='s/.* --output[ =]$[^ ]*$.*/\1/p' 34 | sed_minuso='s/.* -o $[^ ]*$.*/\1/p' 35 | 36 | # In the cases where this matters, 'missing' is being run in the 37 | # srcdir already. 38 | if test -f configure.ac; then 39 | configure_ac=configure.ac 40 | else 41 | configure_ac=configure.in 42 | fi 43 | 44 | msg="missing on your system" 45 | 46 | case $1 in 47 | --run) 48 | # Try to run requested program, and just exit if it succeeds. 49 | run= 50 | shift 51 | "$@" && exit 0 52 | # Exit code 63 means version mismatch. This often happens 53 | # when the user try to use an ancient version of a tool on 54 | # a file that requires a minimum version. In this case we 55 | # we should proceed has if the program had been absent, or 56 | # if --run hadn't been passed. 57 | if test $? = 63; then 58 | run=: 59 | msg="probably too old" 60 | fi 61 | ;; 62 | 63 | -h|--h|--he|--hel|--help) 64 | echo "\ 65 | $0 [OPTION]... PROGRAM [ARGUMENT]... 66 | 67 | Handle 'PROGRAM [ARGUMENT]...' for when PROGRAM is missing, or return an 68 | error status if there is no known handling for PROGRAM. 69 | 70 | Options: 71 | -h, --help display this help and exit 72 | -v, --version output version information and exit 73 | --run try to run the given command, and emulate it if it fails 74 | 75 | Supported PROGRAM values: 76 | aclocal touch file 'aclocal.m4' 77 | autoconf touch file 'configure' 78 | autoheader touch file 'config.h.in' 79 | autom4te touch the output file, or create a stub one 80 | automake touch all 'Makefile.in' files 81 | bison create 'y.tab.[ch]', if possible, from existing .[ch] 82 | flex create 'lex.yy.c', if possible, from existing .c 83 | help2man touch the output file 84 | lex create 'lex.yy.c', if possible, from existing .c 85 | makeinfo touch the output file 86 | yacc create 'y.tab.[ch]', if possible, from existing .[ch] 87 | 88 | Version suffixes to PROGRAM as well as the prefixes 'gnu-', 'gnu', and 89 | 'g' are ignored when checking the name. 90 | 91 | Send bug reports to ." 92 | exit $? 93 | ;; 94 | 95 | -v|--v|--ve|--ver|--vers|--versi|--versio|--version) 96 | echo "missing $scriptversion (GNU Automake)" 97 | exit $? 98 | ;; 99 | 100 | -*) 101 | echo 1>&2 "$0: Unknown '$1' option" 102 | echo 1>&2 "Try '$0 --help' for more information" 103 | exit 1 104 | ;; 105 | 106 | esac 107 | 108 | # normalize program name to check for. 109 | program=`echo "$1" | sed ' 110 | s/^gnu-//; t 111 | s/^gnu//; t 112 | s/^g//; t'` 113 | 114 | # Now exit if we have it, but it failed. Also exit now if we 115 | # don't have it and --version was passed (most likely to detect 116 | # the program). This is about non-GNU programs, so use $1 not 117 | # $program. 118 | case $1 in 119 | lex*|yacc*) 120 | # Not GNU programs, they don't have --version. 121 | ;; 122 | 123 | *) 124 | if test -z "$run" && ($1 --version) > /dev/null 2>&1; then 125 | # We have it, but it failed. 126 | exit 1 127 | elif test "x$2" = "x--version" || test "x$2" = "x--help"; then 128 | # Could not run --version or --help. This is probably someone 129 | # running '$TOOL --version' or '$TOOL --help' to check whether 130 | # $TOOL exists and not knowing $TOOL uses missing. 131 | exit 1 132 | fi 133 | ;; 134 | esac 135 | 136 | # If it does not exist, or fails to run (possibly an outdated version), 137 | # try to emulate it. 138 | case $program in 139 | aclocal*) 140 | echo 1>&2 "\ 141 | WARNING: '$1' is $msg. You should only need it if 142 | you modified 'acinclude.m4' or '${configure_ac}'. You might want 143 | to install the Automake and Perl packages. Grab them from 144 | any GNU archive site." 145 | touch aclocal.m4 146 | ;; 147 | 148 | autoconf*) 149 | echo 1>&2 "\ 150 | WARNING: '$1' is $msg. You should only need it if 151 | you modified '${configure_ac}'. You might want to install the 152 | Autoconf and GNU m4 packages. Grab them from any GNU 153 | archive site." 154 | touch configure 155 | ;; 156 | 157 | autoheader*) 158 | echo 1>&2 "\ 159 | WARNING: '$1' is $msg. You should only need it if 160 | you modified 'acconfig.h' or '${configure_ac}'. You might want 161 | to install the Autoconf and GNU m4 packages. Grab them 162 | from any GNU archive site." 163 | files=`sed -n 's/^[ ]*A[CM]_CONFIG_HEADER($[^)]*$).*/\1/p' ${configure_ac}` 164 | test -z "$files" && files="config.h" 165 | touch_files= 166 | for f in $files; do 167 | case $f in 168 | *:*) touch_files="$touch_files "`echo "$f" | 169 | sed -e 's/^[^:]*://' -e 's/:.*//'`;; 170 | *) touch_files="$touch_files $f.in";; 171 | esac 172 | done 173 | touch $touch_files 174 | ;; 175 | 176 | automake*) 177 | echo 1>&2 "\ 178 | WARNING: '$1' is $msg. You should only need it if 179 | you modified 'Makefile.am', 'acinclude.m4' or '${configure_ac}'. 180 | You might want to install the Automake and Perl packages. 181 | Grab them from any GNU archive site." 182 | find . -type f -name Makefile.am -print | 183 | sed 's/\.am$/.in/' | 184 | while read f; do touch "$f"; done 185 | ;; 186 | 187 | autom4te*) 188 | echo 1>&2 "\ 189 | WARNING: '$1' is needed, but is $msg. 190 | You might have modified some files without having the 191 | proper tools for further handling them. 192 | You can get '$1' as part of Autoconf from any GNU 193 | archive site." 194 | 195 | file=`echo "$*" | sed -n "$sed_output"` 196 | test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"` 197 | if test -f "$file"; then 198 | touch $file 199 | else 200 | test -z "$file" || exec >$file 201 | echo "#! /bin/sh" 202 | echo "# Created by GNU Automake missing as a replacement of" 203 | echo "# $ $@" 204 | echo "exit 0" 205 | chmod +x $file 206 | exit 1 207 | fi 208 | ;; 209 | 210 | bison*|yacc*) 211 | echo 1>&2 "\ 212 | WARNING: '$1' $msg. You should only need it if 213 | you modified a '.y' file. You may need the Bison package 214 | in order for those modifications to take effect. You can get 215 | Bison from any GNU archive site." 216 | rm -f y.tab.c y.tab.h 217 | if test $# -ne 1; then 218 | eval LASTARG=\${$#} 219 | case $LASTARG in 220 | *.y) 221 | SRCFILE=`echo "$LASTARG" | sed 's/y$/c/'` 222 | if test -f "$SRCFILE"; then 223 | cp "$SRCFILE" y.tab.c 224 | fi 225 | SRCFILE=`echo "$LASTARG" | sed 's/y$/h/'` 226 | if test -f "$SRCFILE"; then 227 | cp "$SRCFILE" y.tab.h 228 | fi 229 | ;; 230 | esac 231 | fi 232 | if test ! -f y.tab.h; then 233 | echo >y.tab.h 234 | fi 235 | if test ! -f y.tab.c; then 236 | echo 'main() { return 0; }' >y.tab.c 237 | fi 238 | ;; 239 | 240 | lex*|flex*) 241 | echo 1>&2 "\ 242 | WARNING: '$1' is $msg. You should only need it if 243 | you modified a '.l' file. You may need the Flex package 244 | in order for those modifications to take effect. You can get 245 | Flex from any GNU archive site." 246 | rm -f lex.yy.c 247 | if test $# -ne 1; then 248 | eval LASTARG=\${$#} 249 | case $LASTARG in 250 | *.l) 251 | SRCFILE=`echo "$LASTARG" | sed 's/l$/c/'` 252 | if test -f "$SRCFILE"; then 253 | cp "$SRCFILE" lex.yy.c 254 | fi 255 | ;; 256 | esac 257 | fi 258 | if test ! -f lex.yy.c; then 259 | echo 'main() { return 0; }' >lex.yy.c 260 | fi 261 | ;; 262 | 263 | help2man*) 264 | echo 1>&2 "\ 265 | WARNING: '$1' is $msg. You should only need it if 266 | you modified a dependency of a manual page. You may need the 267 | Help2man package in order for those modifications to take 268 | effect. You can get Help2man from any GNU archive site." 269 | 270 | file=`echo "$*" | sed -n "$sed_output"` 271 | test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"` 272 | if test -f "$file"; then 273 | touch $file 274 | else 275 | test -z "$file" || exec >$file 276 | echo ".ab help2man is required to generate this page" 277 | exit $? 278 | fi 279 | ;; 280 | 281 | makeinfo*) 282 | echo 1>&2 "\ 283 | WARNING: '$1' is $msg. You should only need it if 284 | you modified a '.texi' or '.texinfo' file, or any other file 285 | indirectly affecting the aspect of the manual. The spurious 286 | call might also be the consequence of using a buggy 'make' (AIX, 287 | DU, IRIX). You might want to install the Texinfo package or 288 | the GNU make package. Grab either from any GNU archive site." 289 | # The file to touch is that specified with -o ... 290 | file=`echo "$*" | sed -n "$sed_output"` 291 | test -z "$file" && file=`echo "$*" | sed -n "$sed_minuso"` 292 | if test -z "$file"; then 293 | # ... or it is the one specified with @setfilename ... 294 | infile=`echo "$*" | sed 's/.* $[^ ]*$ *$/\1/'` 295 | file=`sed -n ' 296 | /^@setfilename/{ 297 | s/.* $[^ ]*$ *$/\1/ 298 | p 299 | q 300 | }' $infile` 301 | # ... or it is derived from the source name (dir/f.texi becomes f.info) 302 | test -z "$file" && file=`echo "$infile" | sed 's,.*/,,;s,.[^.]*$,,'`.info 303 | fi 304 | # If the file does not exist, the user really needs makeinfo; 305 | # let's fail without touching anything. 306 | test -f $file || exit 1 307 | touch $file 308 | ;; 309 | 310 | *) 311 | echo 1>&2 "\ 312 | WARNING: '$1' is needed, and is $msg. 313 | You might have modified some files without having the 314 | proper tools for further handling them. Check the 'README' file, 315 | it often tells you about the needed prerequisites for installing 316 | this package. You may also peek at any GNU archive site, in case 317 | some other package would contain this missing '$1' program." 318 | exit 1 319 | ;; 320 | esac 321 | 322 | exit 0 323 | 324 | # Local variables: 325 | # eval: (add-hook 'write-file-hooks 'time-stamp) 326 | # time-stamp-start: "scriptversion=" 327 | # time-stamp-format: "%:y-%02m-%02d.%02H" 328 | # time-stamp-time-zone: "UTC" 329 | # time-stamp-end: "; # UTC" 330 | # End: 331 | -------------------------------------------------------------------------------- /src/option.h: -------------------------------------------------------------------------------- 1 | /** 2 | * options.h 3 | * 4 | * Created on: 2011-7-14 5 | * Author: auxten 6 | **/ 7 | 8 | #ifndef OPTIONS_H_ 9 | #define OPTIONS_H_ 10 | 11 | /// process args for client 12 | int clnt_parse_opt(int argc, char *argv[], s_job_t * jo); 13 | /// process args for server 14 | int serv_parse_opt(int argc, char *argv[]); 15 | 16 | #endif /** OPTIONS_H_ **/ 17 | -------------------------------------------------------------------------------- /src/path.h: -------------------------------------------------------------------------------- 1 | /** 2 | * path.h 3 | * 4 | * Created on: 2011-5-11 5 | * Author: auxten 6 | **/ 7 | 8 | #ifndef PATH_H_ 9 | #define PATH_H_ 10 | 11 | ///file and for file test.eg: flag & GKO_FILE 12 | static const int GKO_FILE = 0001; 13 | ///dir and for dir test 14 | static const int GKO_DIR = 0002; 15 | ///nonexisted and for nonexisted test 16 | static const int GKO_NONE = 0004; 17 | ///other 18 | static const int GKO_OTHR = 0000; 19 | ///symlink to file 20 | static const int GKO_LFILE = 0011; 21 | ///symlink to dir 22 | static const int GKO_LDIR = 0012; 23 | ///symlink to nonexist 24 | static const int GKO_LNONE = 0014; 25 | ///symlink to other 26 | static const int GKO_LOTHR = 0010; 27 | ///for symlink test 28 | static const int GKO_LINK = 0010; 29 | ///error 30 | static const int GKO_ERR = 0100; 31 | 32 | 33 | /// ../path//// TO ../path 34 | int inplace_strip_tailing_slash(char * path); 35 | /// ../path TO ../path/ 36 | int inplace_add_tailing_slash(char * path); 37 | /// get the base name of a string 38 | int get_base_name_index(char * in, const char * out); 39 | /// merge into dir_name/base_name 40 | int merge_path(char * out, const char * dir_name, const char * base_name); 41 | /// change remote path to local path, store it in path 42 | int change_to_local_path(char * path, const char * req_path, 43 | const char * local_path, char dst_path_exist); 44 | /// get the symlink dest's absolute path, store it in abs_path 45 | char * symlink_dest_to_abs_path(char * abs_path, const char * symlink); 46 | /// generate snap file path with requested localpath and requested remote uri 47 | unsigned gen_snap_fpath(char *snap_fpath, const char * localpath, 48 | const char * uri); 49 | /// generate snap file path with requested localpath and requested remote uri 50 | int path_type(const char * path); 51 | /// generate snap file path with requested localpath and requested remote uri 52 | int mk_dir_symlink_file(s_job_t * jo, char * to_continue); 53 | /// correct the file and dir mode, cause for write in we create them with mode|S_IWUSR 54 | int correct_mode(s_job_t * jo); 55 | /// convert the remote path recv from JOIN to local path 56 | int process_path(s_job_t * jo); 57 | 58 | #endif /** PATH_H_ **/ 59 | -------------------------------------------------------------------------------- /src/progress.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * progress.cpp 3 | * 4 | * Created on: 2011-9-12 5 | * Author: auxten 6 | */ 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "gingko.h" 18 | 19 | 20 | ///mutex for up bandwidth limit 21 | static pthread_mutex_t progress_lock = PTHREAD_MUTEX_INITIALIZER; 22 | 23 | extern s_job_t g_job; 24 | 25 | int show_progress(int count) 26 | { 27 | static GKO_INT64 done_count = 0; 28 | 29 | if (count > 0) 30 | { 31 | pthread_mutex_lock(&progress_lock); 32 | done_count += count; 33 | pthread_mutex_unlock(&progress_lock); 34 | } 35 | 36 | if (g_job.block_count) 37 | { 38 | printf("\rDownload progress:\t%5.2f %%", (double)done_count * 100 / g_job.block_count); 39 | } 40 | fflush(stdout); 41 | return 0; 42 | } 43 | 44 | 45 | -------------------------------------------------------------------------------- /src/progress.h: -------------------------------------------------------------------------------- 1 | /* 2 | * progress.h 3 | * 4 | * Created on: 2011-9-12 5 | * Author: auxten 6 | */ 7 | 8 | #ifndef PROGRESS_H_ 9 | #define PROGRESS_H_ 10 | 11 | 12 | int show_progress(int count); 13 | 14 | #endif /* PROGRESS_H_ */ 15 | -------------------------------------------------------------------------------- /src/route.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * route.cpp 3 | * 4 | * Created on: 2011-5-25 5 | * Author: auxten 6 | **/ 7 | #ifndef GINGKO_CLNT 8 | #define GINGKO_CLNT 9 | #endif /** GINGKO_CLNT **/ 10 | 11 | #include 12 | 13 | #include "gingko.h" 14 | #include "hash/xor_hash.h" 15 | #include "snap.h" 16 | #include "progress.h" 17 | #include "gingko_clnt.h" 18 | #include "log.h" 19 | #include "async_pool.h" 20 | 21 | GINGKO_OVERLOAD_S_HOST_EQ 22 | 23 | extern pthread_mutex_t g_blk_hostset_lock; 24 | extern s_gingko_global_t gko; 25 | 26 | /** 27 | * @brief compare by ip distance from the_clnt 28 | * 29 | * @see 30 | * @note 31 | * @author auxten 32 | * @date 2011-8-1 33 | **/ 34 | const bool cmpByDistance(const s_host_t & h1, const s_host_t & h2) 35 | { 36 | ///return abs(h1.port - 59999) > abs(h2.port - 59999); 37 | s_host_t * server = & gko_pool::gko_serv; 38 | return host_distance(server, &h1) 39 | < host_distance(server, &h2); 40 | } 41 | 42 | /** 43 | * @brief get the possible block source, src_max in max 44 | * 45 | * @see 46 | * @note 47 | * @author auxten 48 | * @date 2011-8-1 49 | **/ 50 | int get_blk_src(s_job_t * jo, unsigned src_max, GKO_INT64 blk_idx, 51 | std::vector * h_vec) 52 | { 53 | s_block_t * b; 54 | const GKO_INT64 start_idx = blk_idx; 55 | 56 | (*h_vec).clear(); 57 | GKO_UINT64 max = MIN(src_max + 1, (*jo->host_set).size()); 58 | /// count back blk to find the upper stream src 59 | /// Note that, the src count may be more than the max 60 | while ((*h_vec).size() < max) 61 | { 62 | b = jo->blocks + blk_idx; 63 | pthread_mutex_lock(&g_blk_hostset_lock); 64 | if (b->host_set != NULL && (*(b->host_set)).size() != 0) 65 | { 66 | for (std::set::const_iterator i = (*(b->host_set)).begin(); i 67 | != (*(b->host_set)).end(); i++) 68 | { 69 | if (find((*h_vec).begin(), (*h_vec).end(), *i) 70 | == (*h_vec).end()) 71 | { 72 | (*h_vec).push_back(*i); 73 | } 74 | } 75 | } 76 | pthread_mutex_unlock(&g_blk_hostset_lock); 77 | ///printf("blk_idx: %lld\n", blk_idx); 78 | blk_idx = prev_b(jo, blk_idx); 79 | 80 | /** 81 | * in case of dead lock, when a client quit between 82 | * max is decided and the while loop is started 83 | */ 84 | if (blk_idx == start_idx) 85 | { 86 | break; 87 | } 88 | } 89 | /// sort the src by distance, cause we just need the first src_max+1 90 | sort((*h_vec).begin(), (*h_vec).end(), cmpByDistance); 91 | /// insert gko.the_serv 92 | (*h_vec).push_back(gko.the_serv); 93 | return (*h_vec).size(); 94 | } 95 | 96 | /** 97 | * @brief decide the final src from the host set generated by get_blk_src() 98 | * @brief return block count downloaded during test the source speed 99 | * 100 | * @see 101 | * @return only return 0 or positive num 102 | * @note 103 | * @author auxten 104 | * @date 2011-8-1 105 | **/ 106 | int decide_src(s_job_t * jo, int src_max, GKO_INT64 blk_idx, 107 | std::vector * h_vec, s_host_t * h, char * buf) 108 | { 109 | int num; 110 | int host_i = 0; 111 | int blk_i = 0; 112 | struct timeval before_tv; 113 | struct timeval after_tv; 114 | GKO_INT64 fastest_time = MAX_INT64; 115 | GKO_INT64 tmp_time; 116 | std::vector::iterator fastest = (*h_vec).end(); 117 | s_block_t * b; 118 | 119 | for (std::vector::iterator i = (*h_vec).begin(); 120 | i != (*h_vec).end(); 121 | i++) 122 | { 123 | /** 124 | * if the first host in the vector is myself, pass it 125 | **/ 126 | if (host_distance(&gko_pool::gko_serv, &(*i)) == 0) 127 | { 128 | continue; 129 | } 130 | /** 131 | * do not req the server if there is already a available data src 132 | **/ 133 | if (*i == gko.the_serv) 134 | { 135 | if (fastest_time != MAX_INT64) 136 | { 137 | continue; 138 | } 139 | } 140 | /** 141 | * get one block and calculate the time used 142 | **/ 143 | gettimeofday(&before_tv, NULL); 144 | num = get_blocks_c(jo, &(*i), (blk_idx + blk_i) % jo->block_count, 1, 0 & ~W_DISK, buf); 145 | gettimeofday(&after_tv, NULL); 146 | b = jo->blocks + (blk_idx + blk_i) % jo->block_count; 147 | if (num == 1 && digest_ok(buf, b)) 148 | { 149 | if (UNLIKELY(writeblock(jo, (u_char *) buf, b) < 0)) 150 | { 151 | return 0; 152 | } 153 | else 154 | { 155 | b->done = 1; 156 | dump_progress(jo, b); 157 | } 158 | blk_i++; 159 | tmp_time = (after_tv.tv_sec - before_tv.tv_sec) * 1000000LL 160 | + after_tv.tv_usec - before_tv.tv_usec; 161 | } 162 | else 163 | { 164 | /** 165 | * if get no block, make its time longest 166 | **/ 167 | tmp_time = MAX_INT64; 168 | } 169 | // gko_log(DEBUG, "%u time: %lld", (*i).port, tmp_time); 170 | if (fastest_time > tmp_time) 171 | { 172 | fastest_time = tmp_time; 173 | fastest = i; 174 | } 175 | if (++host_i == src_max) 176 | { 177 | ///make i point to the "gko.the_serv" prev, the ++ will make it serv 178 | i = (*h_vec).end() - 2; 179 | // gko_log(DEBUG, "next: %u", (*i).port); 180 | continue; 181 | } 182 | } 183 | ///get NO available src 184 | if (fastest == (*h_vec).end()) 185 | { 186 | return 0; 187 | } 188 | memset(h, 0, sizeof(s_host_t)); 189 | memcpy(h, &(*fastest), sizeof(s_host_t)); 190 | ///printf("choose %s:%d\n", h->addr, h->port); 191 | return blk_i; 192 | } 193 | -------------------------------------------------------------------------------- /src/route.h: -------------------------------------------------------------------------------- 1 | /** 2 | * route.h 3 | * 4 | * Created on: 2011-5-25 5 | * Author: auxten 6 | **/ 7 | 8 | #ifndef ROUTE_H_ 9 | #define ROUTE_H_ 10 | 11 | /// get the possible block source, src_max in max 12 | int get_blk_src(s_job_t * jo, unsigned src_max, GKO_INT64 blk_idx, 13 | std::vector * h_vec); 14 | /// decide the final src from the host set generated by get_blk_src() 15 | int decide_src(s_job_t * jo, int src_max, GKO_INT64 blk_idx, 16 | std::vector * h_vec, s_host_t * h, char * buf); 17 | 18 | #endif /** ROUTE_H_ **/ 19 | -------------------------------------------------------------------------------- /src/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | start() 4 | { if [ `uname` = 'Linux' ];then 5 | echo "############# tcp_tw_recycle = $(cat /proc/sys/net/ipv4/tcp_tw_recycle)" 6 | echo "############# tcp_tw_reuse = $(cat /proc/sys/net/ipv4/tcp_tw_reuse)" 7 | echo "############# tcp_syncookies = $(cat /proc/sys/net/ipv4/tcp_syncookies)" 8 | echo "############# tcp_fin_timeout = $(cat /proc/sys/net/ipv4/tcp_fin_timeout)" 9 | fi 10 | env CPUPROFILE=/tmp/gkod.prof ../src/gkod 11 | if [ `uname` = 'Linux' ];then 12 | netstat -npl | grep 'gkod' 13 | else 14 | lsof -ni -P | grep 'gkod' 15 | fi 16 | } 17 | 18 | stop() 19 | { 20 | killall gkod 21 | if [ `uname` = 'Linux' ];then 22 | netstat -npl | grep 'gkod' 23 | else 24 | lsof -ni -P | grep 'gkod' 25 | fi 26 | echo "All stoped!" 27 | } 28 | 29 | starttest() 30 | { 31 | ./clnt_unittest 32 | ./serv_unittest 33 | } 34 | 35 | makeit() 36 | { 37 | cd ../src/ && cd hash && make && cd .. && make -j 4 && echo "All made!" 38 | } 39 | 40 | makedbg() 41 | { 42 | cd ../src/ && ./configure CXXFLAGS='-O0 -ggdb' --enable-debug && make clean; cd hash && make && cd .. && make -j 4 43 | } 44 | 45 | makeproduct() 46 | { 47 | cd ../src/ && ./configure CXXFLAGS='-O2 -ggdb' && make clean; cd hash && make && cd .. && make -j 4 48 | } 49 | 50 | makeo3() 51 | { 52 | cd ../src/ && ./configure CXXFLAGS='-O3 -ggdb' --enable-o3 && make clean; cd hash && make && cd .. && make -j 4 53 | } 54 | 55 | makeprofile() 56 | { 57 | cd ../src/ && ./configure CXXFLAGS='-O2 -ggdb' --enable-profile && make clean; cd hash && make && cd .. && make -j 4 58 | } 59 | 60 | makegprof() 61 | { 62 | cd ../src/ && ./configure CXXFLAGS='-O0 -ggdb' --enable-gprofile && make clean; cd hash && make && cd .. && make -j 4 63 | } 64 | 65 | maketest() 66 | { 67 | cd ../src/ && ./configure CXXFLAGS='-O0 -ggdb' --enable-unittest && make clean; cd hash && make && cd .. && make -j 4 68 | } 69 | 70 | case C"$1" in 71 | Cstart) 72 | stop 73 | start 74 | ;; 75 | 76 | Cstop) 77 | stop 78 | ;; 79 | 80 | Cmake) 81 | stop 82 | makeit && start 83 | ;; 84 | 85 | Cdebug) 86 | stop 87 | makedbg && start 88 | ;; 89 | 90 | Cprof) 91 | stop 92 | makeprofile && start 93 | ;; 94 | 95 | Cgprof) 96 | stop 97 | makegprof && start 98 | ;; 99 | 100 | Cprod) 101 | stop 102 | makeproduct && start 103 | ;; 104 | 105 | Co3) 106 | stop 107 | makeo3 && start 108 | ;; 109 | 110 | Ctest) 111 | stop 112 | maketest && starttest 113 | ;; 114 | 115 | C*) 116 | echo "Usage: $0 {start|stop|status|make|debug|gprof|prof|prod|o3|test}" 117 | ;; 118 | esac 119 | -------------------------------------------------------------------------------- /src/run2.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | start() 4 | { 5 | gkod -l ./server.log 6 | } 7 | 8 | stop() 9 | { 10 | killall gkod && sleep 1 11 | if [ `uname` = 'Linux' ];then 12 | netstat -npl 2>/dev/null| grep 'gkod' 13 | else 14 | lsof -ni -P | grep 'gkod' 15 | fi 16 | } 17 | 18 | status() 19 | { 20 | if [ `uname` = 'Linux' ];then 21 | echo "############# tcp_tw_recycle = $(cat /proc/sys/net/ipv4/tcp_tw_recycle)" 22 | echo "############# tcp_tw_reuse = $(cat /proc/sys/net/ipv4/tcp_tw_reuse)" 23 | echo "############# tcp_syncookies = $(cat /proc/sys/net/ipv4/tcp_syncookies)" 24 | echo "############# tcp_fin_timeout = $(cat /proc/sys/net/ipv4/tcp_fin_timeout)" 25 | fi 26 | if [ `uname` = 'Linux' ];then 27 | netstat -npl 2>/dev/null | grep 'gkod' 28 | else 29 | lsof -ni -P | grep 'gkod' 30 | fi 31 | } 32 | 33 | case C"$1" in 34 | Cstart) 35 | stop &>/dev/null && sleep 1 36 | start 37 | sleep 1 38 | status 39 | ;; 40 | 41 | Cstop) 42 | stop 43 | ;; 44 | 45 | Cstatus) 46 | status 47 | ;; 48 | 49 | C*) 50 | echo "Usage: $0 {start|stop|status}" 51 | ;; 52 | esac 53 | -------------------------------------------------------------------------------- /src/seed.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * recurse_dir.cpp 3 | * gingko 4 | * 5 | * Created by Auxten on 11-4-9. 6 | * Copyright 2011 Baidu. All rights reserved. 7 | * 8 | **/ 9 | #ifndef GINGKO_SERV 10 | #define GINGKO_SERV 11 | #endif /** GINGKO_SERV **/ 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #include "gingko.h" 26 | #include "hash/xor_hash.h" 27 | #include "log.h" 28 | #include "path.h" 29 | #include "job_state.h" 30 | 31 | /** 32 | * @brief for use of damn ftw(), we need these TLS(Thread Local Stack) 33 | * @brief global vars for every thread 34 | * 35 | * @see 36 | * @note 37 | * @author auxten 38 | * @date 2011-8-1 39 | **/ 40 | extern pthread_key_t g_dir_key; 41 | 42 | /** 43 | * @brief init the s_file_t array 44 | * 45 | * @see 46 | * @note 47 | * @author auxten 48 | * @date 2011-8-1 49 | **/ 50 | GKO_STATIC_FUNC int init_struct(const char *name, const struct stat *status, int type, 51 | struct FTW * ftw_info) 52 | { 53 | long i = 0; 54 | s_seed_t *p_dir = (s_seed_t *) pthread_getspecific(g_dir_key); 55 | if (!access(name, R_OK)) 56 | { 57 | if (type == FTW_D) 58 | { 59 | (p_dir->files + p_dir->init_s_file_t_iter)->size = -1; 60 | } 61 | else if (type == FTW_SL) 62 | { 63 | (p_dir->files + p_dir->init_s_file_t_iter)->size = -2; 64 | if (-1 == readlink(name, 65 | (p_dir->files + p_dir->init_s_file_t_iter)->sympath, 66 | MAX_PATH_LEN)) 67 | { 68 | gko_log(FATAL, "readlink() error"); 69 | } 70 | } 71 | else if (type == FTW_F && path_type(name) == GKO_FILE) 72 | { 73 | (p_dir->files + p_dir->init_s_file_t_iter)->size = status->st_size; 74 | ///printf("s: %d, e: %d", BLOCK_SIZE, BLOCK_COUNT(tmp_size+status->st_size)); 75 | for (i = p_dir->tmp_size / (long) (BLOCK_SIZE); i 76 | < BLOCK_COUNT(p_dir->tmp_size+status->st_size); i++) 77 | { 78 | if (i - p_dir->last_init_block == 1) 79 | { 80 | (p_dir->blocks + i)->size = BLOCK_SIZE; 81 | (p_dir->blocks + i)->start_f = p_dir->init_s_file_t_iter; 82 | (p_dir->blocks + i)->start_off = i * BLOCK_SIZE 83 | - p_dir->tmp_size; 84 | p_dir->last_init_block = i; 85 | } 86 | } 87 | p_dir->tmp_size += status->st_size; 88 | } 89 | else 90 | { 91 | gko_log(FATAL, "not supported file type for %s", name); 92 | return 0; 93 | } 94 | 95 | // memcpy(&((p_dir->files + p_dir->init_s_file_t_iter)->f_stat), status, 96 | // sizeof(struct stat)); 97 | memcpy(&((p_dir->files + p_dir->init_s_file_t_iter)->mtime), &status->st_mtime, 98 | sizeof(time_t)); 99 | (p_dir->files + p_dir->init_s_file_t_iter)->mode = status->st_mode; 100 | memcpy((p_dir->files + p_dir->init_s_file_t_iter)->name, name, 101 | strlen(name) + 1); 102 | p_dir->init_s_file_t_iter++; 103 | } 104 | else 105 | { 106 | gko_log(FATAL, "have no read permission for %s", name); 107 | } 108 | pthread_setspecific(g_dir_key, p_dir); 109 | return 0; 110 | } 111 | 112 | /** 113 | * @brief count the file num 114 | * 115 | * @see 116 | * @note 117 | * @author auxten 118 | * @date 2011-8-1 119 | **/ 120 | GKO_STATIC_FUNC int file_counter(const char *name, const struct stat *status, int type, 121 | struct FTW * ftw_info) 122 | { 123 | s_seed_t *p_dir = (s_seed_t *) pthread_getspecific(g_dir_key); 124 | if (((type == FTW_F && path_type(name) == GKO_FILE) || type == FTW_SL 125 | || type == FTW_D)) 126 | { 127 | if (!access(name, R_OK)) 128 | { 129 | p_dir->file_count++; 130 | if (type == FTW_F) 131 | { 132 | p_dir->total_size += status->st_size; 133 | } 134 | } 135 | else 136 | { 137 | gko_log(FATAL, "have no read permission for %s", name); 138 | } 139 | ///printf("%ld", total_size); 140 | } 141 | pthread_setspecific(g_dir_key, p_dir); 142 | return 0; 143 | } 144 | 145 | /** 146 | * @brief count the total file size, only file !!! 147 | * 148 | * @see 149 | * @note 150 | * @author auxten 151 | * @date 2011-8-1 152 | **/ 153 | GKO_STATIC_FUNC int init_total_count_size(s_job_t * jo) 154 | { 155 | s_seed_t *p_dir = (s_seed_t *) pthread_getspecific(g_dir_key); 156 | int type = path_type(jo->path); 157 | if ((type & GKO_ERR) || (type & GKO_LINK)) 158 | { 159 | p_dir->file_count = 0; 160 | p_dir->total_size = 0; 161 | jo->job_state = JOB_FILE_TYPE_ERR; 162 | gko_log(FATAL, "source %s is symlink", jo->path); 163 | } 164 | else 165 | { 166 | if (type & GKO_FILE) 167 | { 168 | /** if it's a file or symlink to file **/ 169 | struct stat file_stat; 170 | if(stat(jo->path, &file_stat) != 0) 171 | { 172 | gko_log(FATAL, "stat file %s error", jo->path); 173 | return -1; 174 | } 175 | file_counter(jo->path, &file_stat, FTW_F, NULL); 176 | } 177 | else if (type & GKO_DIR) 178 | { 179 | /** if it's a dir **/ 180 | if(nftw(jo->path, file_counter, MAX_SCAN_DIR_FD, FTW_PHYS) != 0) 181 | { 182 | gko_log(FATAL, "nftw path %s error", jo->path); 183 | return -1; 184 | } 185 | } 186 | else 187 | { 188 | jo->job_state = JOB_FILE_TYPE_ERR; 189 | gko_log(FATAL, "the requested path %s type is special", jo->path); 190 | return -1; 191 | } 192 | } 193 | /** 194 | * init the files_size blocks_size 195 | **/ 196 | GKO_INT64 blk_cnt = BLOCK_COUNT(p_dir->total_size); 197 | jo->total_size = p_dir->total_size; 198 | jo->files_size = p_dir->file_count * sizeof(s_file_t); 199 | jo->blocks_size = blk_cnt * sizeof(s_block_t); 200 | jo->block_count = blk_cnt; 201 | jo->file_count = p_dir->file_count; 202 | 203 | return 0; 204 | } 205 | 206 | /** 207 | * @brief the init_struct caller 208 | * 209 | * @see 210 | * @note 211 | * @author auxten 212 | * @date 2011-8-1 213 | **/ 214 | GKO_STATIC_FUNC int init_seed(s_job_t * jo) 215 | { 216 | s_seed_t *p_dir = (s_seed_t *) pthread_getspecific(g_dir_key); 217 | int type = path_type(jo->path); 218 | if ((type & GKO_ERR) || (type & GKO_LINK)) 219 | { 220 | p_dir->file_count = 0; 221 | p_dir->total_size = 0; 222 | jo->job_state = JOB_FILE_TYPE_ERR; 223 | gko_log(FATAL, "source %s is symlink", jo->path); 224 | } 225 | else 226 | { 227 | if (type & GKO_FILE) 228 | { 229 | /** if it's a file or symlink to file **/ 230 | struct stat file_stat; 231 | if(stat(jo->path, &file_stat) != 0) 232 | { 233 | gko_log(FATAL, "stat file %s error", jo->path); 234 | return -1; 235 | } 236 | init_struct(jo->path, &file_stat, FTW_F, NULL); 237 | } 238 | else if (type & GKO_DIR) 239 | { 240 | if(nftw(jo->path, init_struct, MAX_SCAN_DIR_FD, FTW_PHYS) != 0) 241 | { 242 | gko_log(FATAL, "nftw path %s error", jo->path); 243 | return -1; 244 | } 245 | } 246 | else 247 | { 248 | jo->job_state = JOB_FILE_TYPE_ERR; 249 | gko_log(FATAL, "the requested path %s type is special", jo->path); 250 | return -1; 251 | } 252 | } 253 | for (int i = 0; i < BLOCK_COUNT(p_dir->total_size); i++) 254 | { 255 | ///init the block.done 256 | (p_dir->blocks + i)->done = 1; 257 | } 258 | 259 | return 0; 260 | } 261 | 262 | 263 | /** 264 | * @brief init the s_job_t s_block_t s_file_t struct for the job 265 | * 266 | * @see 267 | * @note 268 | * @author auxten 269 | * @date 2011-8-1 270 | **/ 271 | int recurse_dir(s_job_t * jo) 272 | { 273 | s_seed_t dir; 274 | s_seed_t * p_dir = &dir; 275 | p_dir->file_count = 0; 276 | p_dir->init_s_file_t_iter = 0; 277 | p_dir->init_s_block_t_iter = 0; 278 | p_dir->total_size = 0; 279 | p_dir->tmp_size = 0; 280 | p_dir->last_init_block = -1; 281 | pthread_setspecific(g_dir_key, p_dir); 282 | p_dir = (s_seed_t *) pthread_getspecific(g_dir_key); 283 | 284 | /** 285 | * as usage of global static, get_file_count can only run once in one proc 286 | **/ 287 | GKO_INT64 blk_cnt; 288 | GKO_INT64 min_full_blk_cnt; 289 | gettimeofday(&jo->dl_time, NULL); 290 | if(init_total_count_size(jo) != 0) 291 | { 292 | gko_log(FATAL, "init_total_count_size error"); 293 | return -1; 294 | } 295 | blk_cnt = BLOCK_COUNT(p_dir->total_size); 296 | min_full_blk_cnt = p_dir->total_size ? (blk_cnt - 1) : 0; 297 | 298 | if (p_dir->file_count <= 1) 299 | { 300 | ///the only file is the dir 301 | gko_log(NOTICE, "path: %s, p_dir->file_count is %lld", jo->path, 302 | p_dir->file_count); 303 | } 304 | if (p_dir->file_count) 305 | { 306 | p_dir->files = new s_file_t[p_dir->file_count]; 307 | } 308 | if (! p_dir->files) 309 | { 310 | gko_log(FATAL, "new for files failed"); 311 | return -1; 312 | } 313 | memset(p_dir->files, 0, sizeof(s_file_t) * p_dir->file_count); 314 | if (p_dir->total_size) 315 | { 316 | p_dir->blocks = new s_block_t[blk_cnt]; 317 | } 318 | if (! p_dir->blocks) 319 | { 320 | gko_log(FATAL, "new for blocks failed"); 321 | return -1; 322 | } 323 | memset(p_dir->blocks, 0, sizeof(s_block_t) * blk_cnt); 324 | gko_log(NOTICE, "s_file_t size: %lu", sizeof(s_file_t)); 325 | gko_log(NOTICE, "file count: %lld", p_dir->file_count); 326 | gko_log(NOTICE, "total size: %lld", p_dir->total_size); 327 | gko_log(NOTICE, "block count: %lld", blk_cnt); 328 | 329 | /** 330 | * init the files and blocks, and init the last block 331 | **/ 332 | if(init_seed(jo) != 0) 333 | { 334 | gko_log(FATAL, "init seed error"); 335 | return -1; 336 | } 337 | 338 | jo->files = p_dir->files; 339 | if (p_dir->total_size) 340 | { 341 | (p_dir->blocks + min_full_blk_cnt)->size = p_dir->total_size 342 | - min_full_blk_cnt * BLOCK_SIZE; 343 | 344 | GKO_INT64 i = p_dir->total_size; 345 | GKO_INT64 j = p_dir->file_count - 1; 346 | GKO_INT64 file_size_tmp = (p_dir->files + j)->size; 347 | for (; i - MAX(file_size_tmp, 0) > min_full_blk_cnt * BLOCK_SIZE && j 348 | >= 0; i -= MAX(file_size_tmp, 0), j--) 349 | { 350 | file_size_tmp = (p_dir->files + j)->size; 351 | ///printf("start_f: %d, off: %d", j, i); 352 | } 353 | 354 | /** 355 | * check the last block 356 | **/ 357 | if (((p_dir->blocks + min_full_blk_cnt)->start_f != j) 358 | || ((p_dir->blocks + min_full_blk_cnt)->start_off 359 | != (min_full_blk_cnt) * BLOCK_SIZE - (i - (p_dir->files 360 | + j)->size))) 361 | { 362 | gko_log(NOTICE, "file division maybe error!"); 363 | } 364 | jo->blocks = p_dir->blocks; 365 | 366 | } 367 | 368 | for (GKO_INT64 i = 0; i < p_dir->file_count; i++) 369 | { 370 | gko_log(NOTICE, "0%3o\t\t%lld\t\t\t%s\t%s", 371 | (p_dir->files + i)->mode & 0777, (p_dir->files + i)->size, 372 | (p_dir->files + i)->name, (p_dir->files + i)->sympath); 373 | } 374 | 375 | return 0; 376 | } 377 | -------------------------------------------------------------------------------- /src/seed.h: -------------------------------------------------------------------------------- 1 | /** 2 | * seed.h 3 | * 4 | * Created on: 2011-7-18 5 | * Author: auxten 6 | **/ 7 | #include "gingko.h" 8 | 9 | #ifndef SEED_H_ 10 | #define SEED_H_ 11 | 12 | /// init the s_job_t s_block_t s_file_t struct for the job 13 | int recurse_dir(s_job_t * jo); 14 | #endif /** SEED_H_ **/ 15 | -------------------------------------------------------------------------------- /src/serv_main.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * serv_main.cpp 3 | * 4 | * Created on: 2011-8-15 5 | * Author: auxten 6 | */ 7 | 8 | 9 | #ifndef GINGKO_SERV 10 | #define GINGKO_SERV 11 | #endif /** GINGKO_SERV **/ 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #ifdef __APPLE__ 35 | #include 36 | #elif defined (__FreeBSD__) 37 | #include 38 | #include 39 | #include 40 | #else 41 | #include 42 | #endif /** __APPLE__ **/ 43 | 44 | #include 45 | #include 46 | #include 47 | #include 48 | 49 | #include "gingko.h" 50 | #include "async_pool.h" 51 | #include "hash/xor_hash.h" 52 | #include "path.h" 53 | #include "log.h" 54 | #include "seed.h" 55 | #include "socket.h" 56 | #include "option.h" 57 | #include "job_state.h" 58 | #include "gingko_serv.h" 59 | 60 | using namespace std; 61 | 62 | /************** PTHREAD STUFF **************/ 63 | ///server wide lock 64 | pthread_mutex_t g_grand_lock; 65 | ///job specific lock 66 | s_lock_t g_job_lock[MAX_JOBS]; 67 | ///for TLS 68 | pthread_key_t g_dir_key; 69 | /************** PTHREAD STUFF **************/ 70 | 71 | /************** FUNC DICT **************/ 72 | #include "gingko_common.h" 73 | /************** FUNC DICT **************/ 74 | 75 | GINGKO_OVERLOAD_S_HOST_LT 76 | 77 | /// jobs map 78 | map g_m_jobs; 79 | 80 | /// gingko global stuff 81 | s_gingko_global_t gko; 82 | 83 | 84 | /** 85 | * @brief daemonlize 86 | * 87 | * @see 88 | * @note 89 | * @author auxten 90 | * @date 2011-8-1 91 | **/ 92 | GKO_STATIC_FUNC int init_daemon(void) 93 | { 94 | int fd; 95 | switch (fork()) 96 | { 97 | case -1: 98 | return (-1); 99 | break; 100 | 101 | case 0: 102 | break; 103 | 104 | default: 105 | _exit(EXIT_SUCCESS); 106 | break; 107 | } 108 | 109 | if (setsid() == -1) 110 | { 111 | return (-1); 112 | } 113 | 114 | if ((fd = open("/dev/null", O_RDWR, 0)) != -1) 115 | { 116 | if (dup2(fd, STDIN_FILENO) < 0) 117 | { 118 | gko_log(FATAL, "dup2 stdin"); 119 | return (-1); 120 | } 121 | if (dup2(fd, STDOUT_FILENO) < 0) 122 | { 123 | gko_log(FATAL, "dup2 stdout"); 124 | return (-1); 125 | } 126 | if (dup2(fd, STDERR_FILENO) < 0) 127 | { 128 | gko_log(FATAL, "dup2 stderr"); 129 | return (-1); 130 | } 131 | 132 | if (fd > STDERR_FILENO) 133 | { 134 | if (close(fd) < 0) 135 | { 136 | gko_log(FATAL, "close /dev/null failed"); 137 | return (-1); 138 | } 139 | } 140 | } 141 | 142 | return 0; 143 | } 144 | 145 | /** 146 | * @brief pthread stuff init 147 | * 148 | * @see 149 | * @note 150 | * @author auxten 151 | * @date 2011-8-1 152 | **/ 153 | GKO_STATIC_FUNC int pthread_init() 154 | { 155 | if (pthread_key_create(&g_dir_key, NULL) != 0) 156 | { 157 | gko_log(FATAL, FLF("pthread_key_create error")); 158 | return -1; 159 | } 160 | if (pthread_mutex_init(&g_grand_lock, NULL) != 0) 161 | { 162 | gko_log(FATAL, FLF("pthread_mutex_init error")); 163 | return -1; 164 | } 165 | for (int i = 0; i < MAX_JOBS; i++) 166 | { 167 | g_job_lock[i].state = LK_FREE; 168 | if (pthread_mutex_init(&(g_job_lock[i].lock), NULL) != 0) 169 | { 170 | gko_log(FATAL, FLF("pthread_mutex_init error")); 171 | return -1; 172 | } 173 | } 174 | return 0; 175 | } 176 | 177 | /** 178 | * @brief pthread stuff clean 179 | * 180 | * @see 181 | * @note 182 | * @author auxten 183 | * @date 2011-8-1 184 | **/ 185 | GKO_STATIC_FUNC int pthread_clean() 186 | { 187 | for (int i = 0; i < MAX_JOBS; i++) 188 | { 189 | if (pthread_mutex_destroy(&(g_job_lock[i].lock)) != 0) 190 | { 191 | gko_log(FATAL, FLF("pthread_mutex_init error")); 192 | return -1; 193 | } 194 | } 195 | if (pthread_mutex_destroy(&g_grand_lock) != 0) 196 | { 197 | gko_log(FATAL, FLF("pthread_mutex_init error")); 198 | return -1; 199 | } 200 | if (pthread_key_delete(g_dir_key) != 0) 201 | { 202 | gko_log(FATAL, FLF("pthread_mutex_init error")); 203 | return -1; 204 | } 205 | 206 | return 0; 207 | } 208 | 209 | /** 210 | * @brief server INT handeler 211 | * 212 | * @see 213 | * @note 214 | * @author auxten 215 | * @date 2011-8-1 216 | **/ 217 | GKO_STATIC_FUNC void serv_int_handler(const int sig) 218 | { 219 | gko.sig_flag = sig; 220 | return; 221 | } 222 | 223 | /** 224 | * @brief this thread will watch the gko.sig_flag and take action 225 | * 226 | * @see 227 | * @note 228 | * @author auxten 229 | * @date 2011-8-9 230 | **/ 231 | GKO_STATIC_FUNC void * serv_int_worker(void * a) 232 | { 233 | while(1) 234 | { 235 | if (UNLIKELY(gko.sig_flag)) 236 | { 237 | gko_log(WARNING, "SIGNAL handled, server terminated"); 238 | /// Clear all status 239 | gko_pool::getInstance()->conn_close(); 240 | gko_quit(2); 241 | } 242 | usleep(CK_SIG_INTERVAL); 243 | } 244 | } 245 | 246 | /** 247 | * @brief server global stuff init also the args processing 248 | * 249 | * @see 250 | * @note 251 | * @author auxten 252 | * @date 2011-8-1 253 | **/ 254 | GKO_STATIC_FUNC int gingko_serv_global_init(int argc, char *argv[]) 255 | { 256 | memset(&gko, 0, sizeof(gko)); 257 | if (serv_parse_opt(argc, argv) == 0) 258 | { 259 | gko_log(NOTICE, "opts parsed successfully"); 260 | } 261 | else 262 | { 263 | return -1; 264 | } 265 | umask(0); 266 | 267 | if (check_ulimit() != 0) 268 | { 269 | return -1; 270 | } 271 | 272 | // // forbid gkod run in root 273 | // if (geteuid() == 0) { 274 | // gko_log(FATAL, "gkod should not run with root!"); 275 | // return -1; 276 | // } 277 | 278 | if (gko.opt.daemon_mode) 279 | { 280 | if (init_daemon()) 281 | { 282 | gko_log(FATAL, "init_daemon failed"); 283 | return -1; 284 | } 285 | } 286 | set_sig(serv_int_handler); 287 | /** 288 | * init global vars stuff 289 | **/ 290 | gko.ready_to_serv = 1; 291 | gko.sig_flag = 0; 292 | 293 | return 0; 294 | } 295 | 296 | /** 297 | * @brief server side async server starter 298 | * 299 | * @see 300 | * @note 301 | * @author auxten 302 | * @date 2011-8-1 303 | **/ 304 | int gingko_serv_async_server() 305 | { 306 | gko_pool * gingko = gko_pool::getInstance(); 307 | gingko->setPort(gko.opt.port); 308 | gingko->setOption(&gko.opt); 309 | gingko->setFuncTable(g_cmd_list, g_func_list_s, CMD_COUNT); 310 | 311 | return gingko->gko_run(); 312 | } 313 | 314 | /** 315 | * @brief main 316 | * 317 | * @see 318 | * @note 319 | * @author auxten 320 | * @date 2011-8-1 321 | **/ 322 | int main(int argc, char *argv[]) 323 | { 324 | ///init_daemon(); 325 | if(gingko_serv_global_init(argc, argv)) 326 | { 327 | gko_log(FATAL, "gingko_serv_global_init failed"); 328 | fprintf(stderr, "Server error, quited\n"); 329 | gko_quit(1); 330 | } 331 | 332 | gko_log(DEBUG, "Debug mode start, i will print tons of log :p!"); 333 | 334 | if (pthread_init() != 0) 335 | { 336 | gko_log(FATAL, FLF("pthread_init error")); 337 | fprintf(stderr, "Server error, quited\n"); 338 | gko_quit(1); 339 | } 340 | if (sig_watcher(serv_int_worker)) 341 | { 342 | gko_log(FATAL, "signal watcher start error"); 343 | fprintf(stderr, "Server error, quited\n"); 344 | gko_quit(1); 345 | } 346 | if (gingko_serv_async_server() != 0) 347 | { 348 | gko_log(WARNING, "gingko_serv_async_server error"); 349 | gko_quit(1); 350 | } 351 | /// basically it will never run here... 352 | pthread_clean(); 353 | return 0; 354 | } 355 | 356 | 357 | -------------------------------------------------------------------------------- /src/serv_unittest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * serv_unittest.cpp 3 | * 4 | * Created on: 2011-8-15 5 | * Author: auxten 6 | */ 7 | 8 | #ifndef GINGKO_SERV 9 | #define GINGKO_SERV 10 | #endif /** GINGKO_SERV **/ 11 | 12 | #include "gingko.h" 13 | #include "async_pool.h" 14 | #include "hash/xor_hash.h" 15 | #include "path.h" 16 | #include "log.h" 17 | #include "seed.h" 18 | #include "socket.h" 19 | #include "option.h" 20 | #include "job_state.h" 21 | #include "gingko_serv.h" 22 | 23 | #define UNITTEST 24 | /************** PTHREAD STUFF **************/ 25 | ///server wide lock 26 | pthread_mutex_t g_grand_lock; 27 | ///job specific lock 28 | s_lock_t g_job_lock[MAX_JOBS]; 29 | pthread_key_t g_dir_key; 30 | /************** PTHREAD STUFF **************/ 31 | /// jobs map 32 | std::map g_m_jobs; 33 | 34 | /// gingko global stuff 35 | s_gingko_global_t gko; 36 | 37 | #ifdef UNITTEST 38 | 39 | #include "gtest/gtest.h" 40 | #include "unittest.h" 41 | 42 | /************** FUNC DICT **************/ 43 | #include "gingko_common.h" 44 | /************** FUNC DICT **************/ 45 | 46 | GINGKO_OVERLOAD_S_HOST_LT 47 | 48 | 49 | TEST(inplace_strip_tailing_slash, with1slash) 50 | { 51 | char test_path1[MAX_PATH_LEN]; 52 | strncpy(test_path1, "aaa/", MAX_PATH_LEN); 53 | char * test_rslt = "aaa"; 54 | inplace_strip_tailing_slash(test_path1); 55 | EXPECT_STREQ(test_rslt, test_path1); 56 | } 57 | 58 | /** 59 | * @brief process args for server 60 | * 61 | * @see 62 | * @note 63 | * @author auxten 64 | * @date 2011-8-1 65 | **/ 66 | //int serv_parse_opt(int argc, char *argv[]) 67 | TEST(serv_parse_opt, serv_parse_opt) 68 | { 69 | const int max_argc = 30; 70 | char argv_test_buf[max_argc][MAX_PATH_LEN] = 71 | { 72 | "gkod", 73 | "-l", "./serv.log", 74 | "-u", "17", "-r", "19", 75 | "-t", "21", "-n", "22", "-b", "127.0.0.1", 76 | "-p", "2121", "--debug" 77 | }; 78 | char * argv_test[max_argc]; 79 | for (int i = 0; i < max_argc; i++) 80 | { 81 | argv_test[i] = (char *)argv_test_buf[i]; 82 | } 83 | int argc_test = 16; 84 | EXPECT_EQ(0, serv_parse_opt(argc_test, argv_test)); 85 | //EXPECT_EQ(htonl(INADDR_LOOPBACK), gko.opt.bind_ip); 86 | EXPECT_EQ(22, gko.opt.connlimit); 87 | EXPECT_EQ(17*1024*1024, gko.opt.limit_up_rate); 88 | EXPECT_EQ(19*1024*1024, gko.opt.limit_disk_r_rate); 89 | EXPECT_EQ(0, gko.opt.need_help); 90 | EXPECT_EQ(0, gko.opt.daemon_mode); 91 | EXPECT_EQ(2121, gko.opt.port); 92 | EXPECT_EQ(1, gko.opt.to_debug); 93 | EXPECT_EQ(21, gko.opt.worker_thread); 94 | EXPECT_STREQ("./serv.log", gko.opt.logpath); 95 | 96 | } 97 | 98 | TEST(serv_show_version, serv_show_version) 99 | { 100 | EXPECT_NO_FATAL_FAILURE(serv_show_version()); 101 | } 102 | 103 | TEST(serv_show_help, serv_show_help) 104 | { 105 | EXPECT_NO_FATAL_FAILURE(serv_show_help()); 106 | } 107 | 108 | 109 | /// test thread pool 110 | TEST(async_threads, gko_pool) 111 | { 112 | gko.ready_to_serv = 1; 113 | gko.sig_flag = 0; 114 | 115 | gko_log(DEBUG, "Debug mode start, i will print tons of log :p!"); 116 | 117 | // if (pthread_init() != 0) 118 | // { 119 | // gko_log(FATAL, FLF("pthread_init error")); 120 | // fprintf(stderr, "Server error, quited\n"); 121 | // gko_quit(1); 122 | // } 123 | 124 | gko_pool * gingko = gko_pool::getInstance(); 125 | EXPECT_NE((gko_pool*)NULL, gingko); 126 | EXPECT_NO_FATAL_FAILURE(gingko->setPort(2120)); 127 | EXPECT_NO_FATAL_FAILURE(gingko->setOption(&gko.opt)); 128 | EXPECT_NO_FATAL_FAILURE( 129 | gingko->setFuncTable(g_cmd_list, g_func_list_s, CMD_COUNT)); 130 | 131 | EXPECT_NO_FATAL_FAILURE(gingko->gko_loopexit(2)); 132 | EXPECT_NO_FATAL_FAILURE(gingko->gko_run()); 133 | } 134 | 135 | /** 136 | * @brief server unittest main 137 | * 138 | * @see 139 | * @note 140 | * @author auxten 141 | * @date 2011-8-15 142 | **/ 143 | int main(int argc, char** argv) 144 | { 145 | strncpy(gko.opt.logpath, "/dev/stdout", MAX_PATH_LEN); 146 | ::testing::InitGoogleTest(&argc, argv); 147 | 148 | return RUN_ALL_TESTS(); 149 | } 150 | #else 151 | int main() 152 | { 153 | 154 | } 155 | #endif /* UNITTEST */ 156 | -------------------------------------------------------------------------------- /src/snap.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * snap.cpp 3 | * 4 | * Created on: 2011-7-14 5 | * Author: auxten 6 | **/ 7 | #ifndef GINGKO_CLNT 8 | #define GINGKO_CLNT 9 | #endif /** GINGKO_CLNT **/ 10 | 11 | #include "gingko.h" 12 | #include "log.h" 13 | 14 | extern s_gingko_global_t gko; 15 | 16 | /** 17 | * @brief dump the progress of this blk 18 | * 19 | * @see 20 | * @note 21 | * @author auxten 22 | * @date 2011-8-1 23 | **/ 24 | void dump_progress(s_job_t * jo, s_block_t * blk) 25 | { 26 | /** 27 | * frist of all fsync the wrote block 28 | * use pwrite to dump every blk write progress 29 | **/ 30 | s_snap_t snap = {'\0'}; 31 | snap.digest = blk->digest; 32 | snap.done = 1; 33 | 34 | if (gko.snap_fd < 0) 35 | { 36 | if ((gko.snap_fd = open(gko.snap_fpath, SNAP_OPEN_FLAG, SNAP_FILE_MODE)) 37 | < 0) 38 | { 39 | gko_log(WARNING, "open snap file failed"); 40 | return; 41 | } 42 | } 43 | 44 | /** 45 | * calculate the offset of block in snap file 46 | **/ 47 | off_t offset = (blk - jo->blocks) * sizeof(s_snap_t); 48 | if (FAIL_CHECK(pwrite(gko.snap_fd, &snap, sizeof(s_snap_t), offset) == -1)) 49 | { 50 | gko_log(WARNING, "pwrite snap failed"); 51 | } 52 | fsync(gko.snap_fd); 53 | return; 54 | } 55 | 56 | /** 57 | * @brief examine the snap file handled by fd, if OK load it 58 | * 59 | * @see 60 | * @note 61 | * @author auxten 62 | * @date 2011-8-1 63 | **/ 64 | int load_snap(s_job_t * jo) 65 | { 66 | struct stat snap_fstat; 67 | 68 | if (gko.snap_fd < 0) 69 | { 70 | if ((gko.snap_fd = open(gko.snap_fpath, SNAP_OPEN_FLAG, SNAP_FILE_MODE)) 71 | < 0) 72 | { 73 | gko_log(WARNING, "open snap file failed"); 74 | return -1; 75 | } 76 | } 77 | 78 | /** 79 | * first check the snap file size and the blk count * s_snap_t 80 | **/ 81 | if (fstat(gko.snap_fd, &snap_fstat) < 0) 82 | { 83 | gko_log(WARNING, "fstat snap file failed"); 84 | return -1; 85 | } 86 | if ((GKO_UINT64) snap_fstat.st_size != jo->block_count * sizeof(s_snap_t)) 87 | { 88 | gko_log(WARNING, "snap file size mismatch"); 89 | return -1; 90 | } 91 | /** 92 | * check the existed file size 93 | **/ 94 | /// this is done in mk_dir_symlink_file() 95 | 96 | /** 97 | * read the snap file 98 | **/ 99 | char * snap_buf; 100 | if (readfileall(gko.snap_fd, 0, snap_fstat.st_size, &snap_buf)) 101 | { 102 | gko_log(WARNING, "snap file read error"); 103 | return -1; 104 | } 105 | /** 106 | * compare digest and set block done 107 | **/ 108 | s_snap_t * snap_p = (s_snap_t *) snap_buf; 109 | s_block_t * blk_p = jo->blocks; 110 | for (int i = 0; i < jo->block_count; i++) 111 | { 112 | if ((blk_p + i)->digest == (snap_p + i)->digest && (snap_p + i)->done) 113 | { 114 | gko_log(NOTICE, "block %d downloaded previously", i); 115 | (blk_p + i)->done = 1; 116 | } 117 | } 118 | delete [] snap_buf; 119 | snap_buf = NULL; 120 | 121 | return 0; 122 | } 123 | -------------------------------------------------------------------------------- /src/snap.h: -------------------------------------------------------------------------------- 1 | /** 2 | * snap.h 3 | * 4 | * Created on: 2011-7-14 5 | * Author: auxten 6 | **/ 7 | 8 | #ifndef SNAP_H_ 9 | #define SNAP_H_ 10 | 11 | /// dump the progress of this blk 12 | void dump_progress(s_job_t * jo, s_block_t * blk); 13 | /// examine the snap file handled by fd, if OK load it 14 | int load_snap(s_job_t * jo); 15 | 16 | #endif /** SNAP_H_ **/ 17 | -------------------------------------------------------------------------------- /src/socket.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * socket.cpp 3 | * 4 | * Created on: 2011-7-17 5 | * Author: auxten 6 | **/ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "config.h" 14 | #include "gingko.h" 15 | #include "log.h" 16 | #include "socket.h" 17 | #ifdef __APPLE__ 18 | #include 19 | #else 20 | #include 21 | #endif /** __APPLE__ **/ 22 | 23 | 24 | /** 25 | * @brief Set non-blocking 26 | * 27 | * @see 28 | * @note 29 | * @author auxten 30 | * @date 2011-8-1 31 | **/ 32 | int setnonblock(int fd) 33 | { 34 | int flags; 35 | 36 | flags = fcntl(fd, F_GETFL); 37 | if (flags < 0) 38 | { 39 | return flags; 40 | } 41 | 42 | if (!(flags & O_NONBLOCK)) 43 | { 44 | flags |= O_NONBLOCK; 45 | if (fcntl(fd, F_SETFL, flags) < 0) 46 | { 47 | return -1; 48 | } 49 | } 50 | return 0; 51 | } 52 | 53 | /** 54 | * @brief Set blocking 55 | * 56 | * @see 57 | * @note 58 | * @author auxten 59 | * @date 2011-8-1 60 | **/ 61 | int setblock(int fd) 62 | { 63 | int flags; 64 | 65 | flags = fcntl(fd, F_GETFL); 66 | if (flags < 0) 67 | { 68 | return flags; 69 | } 70 | 71 | if (flags & O_NONBLOCK) 72 | { 73 | flags &= ~O_NONBLOCK; 74 | if (fcntl(fd, F_SETFL, flags) < 0) 75 | { 76 | return -1; 77 | } 78 | } 79 | return 0; 80 | } 81 | 82 | /** 83 | * @brief connect to a host 84 | * 85 | * @see 86 | * @note 87 | * h: pointer to s_host_t 88 | * recv_sec: receive timeout seconds, 0 for never timeout 89 | * return the socket when succ 90 | * return < 0 when error, specially HOST_DOWN_FAIL indicate host dead 91 | * @author auxten 92 | * @date 2011-8-1 93 | **/ 94 | int connect_host(const s_host_t * h, const int recv_sec, const int send_sec) 95 | { 96 | int sock = -1; 97 | int ret; 98 | int select_ret; 99 | int res; 100 | socklen_t res_size = sizeof res; 101 | struct sockaddr_in channel; 102 | in_addr_t host; 103 | int addr_len; 104 | struct timeval recv_timeout; 105 | struct timeval send_timeout; 106 | #if HAVE_POLL 107 | #else 108 | fd_set wset; 109 | #endif /* HAVE_POLL */ 110 | 111 | addr_len = getaddr_my(h->addr, &host); 112 | if (FAIL_CHECK(!addr_len)) 113 | { 114 | gko_log(WARNING, "gethostbyname %s error", h->addr); 115 | ret = -1; 116 | goto CONNECT_END; 117 | } 118 | sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); 119 | if (FAIL_CHECK(sock < 0)) 120 | { 121 | gko_log(WARNING, "get socket error"); 122 | ret = -1; 123 | goto CONNECT_END; 124 | } 125 | 126 | recv_timeout.tv_usec = 0; 127 | recv_timeout.tv_sec = recv_sec ? recv_sec : RCV_TIMEOUT; 128 | send_timeout.tv_usec = 0; 129 | send_timeout.tv_sec = send_sec ? send_sec : SND_TIMEOUT; 130 | 131 | memset(&channel, 0, sizeof(channel)); 132 | channel.sin_family = AF_INET; 133 | memcpy(&channel.sin_addr.s_addr, &host, addr_len); 134 | channel.sin_port = htons(h->port); 135 | 136 | /** set the connect non-blocking then blocking for add timeout on connect **/ 137 | if (FAIL_CHECK(setnonblock(sock) < 0)) 138 | { 139 | gko_log(WARNING, "set socket non-blocking error"); 140 | ret = -1; 141 | goto CONNECT_END; 142 | } 143 | 144 | /** connect and send the msg **/ 145 | if (FAIL_CHECK(connect(sock, (struct sockaddr *) &channel, sizeof(channel)) && 146 | errno != EINPROGRESS)) 147 | { 148 | gko_log(WARNING, "connect error"); 149 | ret = HOST_DOWN_FAIL; 150 | goto CONNECT_END; 151 | } 152 | 153 | /** Wait for write bit to be set **/ 154 | #if HAVE_POLL 155 | { 156 | struct pollfd pollfd; 157 | 158 | pollfd.fd = sock; 159 | pollfd.events = POLLOUT; 160 | 161 | /* send_sec is in seconds, timeout in ms */ 162 | select_ret = poll(&pollfd, 1, (int)(send_sec * 1000 + 1)); 163 | } 164 | #else 165 | { 166 | FD_ZERO(&wset); 167 | FD_SET(sock, &wset); 168 | select_ret = select(sock + 1, 0, &wset, 0, &send_timeout); 169 | } 170 | #endif /* HAVE_POLL */ 171 | if (select_ret < 0) 172 | { 173 | gko_log(WARNING, "select/poll error on connect"); 174 | ret = HOST_DOWN_FAIL; 175 | goto CONNECT_END; 176 | } 177 | if (!select_ret) 178 | { 179 | gko_log(WARNING, "connect timeout on connect"); 180 | ret = HOST_DOWN_FAIL; 181 | goto CONNECT_END; 182 | } 183 | 184 | /** 185 | * check if connection is RESETed, maybe this is the 186 | * best way to do that 187 | * SEE: http://cr.yp.to/docs/connect.html 188 | **/ 189 | (void) getsockopt(sock, SOL_SOCKET, SO_ERROR, &res, &res_size); 190 | if (CONNECT_DEST_DOWN(res)) 191 | { 192 | // gko_log(NOTICE, "connect dest is down errno: %d", res); 193 | ret = HOST_DOWN_FAIL; 194 | goto CONNECT_END; 195 | } 196 | 197 | ///gko_log(WARNING, "selected %d ret %d, time %d", sock, select_ret, send_timeout.tv_sec); 198 | /** set back blocking **/ 199 | if (FAIL_CHECK(setblock(sock) < 0)) 200 | { 201 | gko_log(WARNING, "set socket non-blocking error"); 202 | ret = -1; 203 | goto CONNECT_END; 204 | } 205 | 206 | /** set recv & send timeout **/ 207 | if (FAIL_CHECK(setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (char *) &recv_timeout, 208 | sizeof(struct timeval)))) 209 | { 210 | gko_log(WARNING, "setsockopt SO_RCVTIMEO error"); 211 | ret = -1; 212 | goto CONNECT_END; 213 | } 214 | if (FAIL_CHECK(setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (char *) &send_timeout, 215 | sizeof(struct timeval)))) 216 | { 217 | gko_log(WARNING, "setsockopt SO_SNDTIMEO error"); 218 | ret = -1; 219 | goto CONNECT_END; 220 | } 221 | 222 | ret = sock; 223 | 224 | CONNECT_END: 225 | /// 226 | if (ret < 0 && sock >= 0) 227 | { 228 | close_socket(sock); 229 | } 230 | return ret; 231 | } 232 | 233 | /** 234 | * @brief gracefully close a socket, for client side 235 | * 236 | * @see 237 | * @note 238 | * @author auxten 239 | * @date 2011-8-1 240 | **/ 241 | int close_socket(int sock) 242 | { 243 | if (FAIL_CHECK(close(sock))) 244 | { 245 | gko_log(WARNING, "close sock error"); 246 | return -1; 247 | } 248 | return 0; 249 | } 250 | 251 | -------------------------------------------------------------------------------- /src/socket.h: -------------------------------------------------------------------------------- 1 | /** 2 | * socket.h 3 | * 4 | * Created on: 2011-7-17 5 | * Author: auxten 6 | **/ 7 | 8 | #ifndef SOCKET_H_ 9 | #define SOCKET_H_ 10 | 11 | /// Set non-blocking 12 | int setnonblock(int fd); 13 | /// Set blocking 14 | int setblock(int fd); 15 | /// gracefully close a socket, for client side 16 | int close_socket(int sock); 17 | /// connect to a host 18 | int connect_host(const s_host_t * h, const int recv_sec, const int send_sec); 19 | 20 | #endif /** SOCKET_H_ **/ 21 | -------------------------------------------------------------------------------- /src/unittest.h: -------------------------------------------------------------------------------- 1 | /* 2 | * unittest.h 3 | * 4 | * Created on: 2011-8-16 5 | * Author: auxten 6 | */ 7 | 8 | #ifdef UNITTEST 9 | ///async_conn.cpp (3 matches) 10 | int conn_client_list_init(); 11 | int gingko_serv_async_server_base_init(); 12 | int gingko_clnt_async_server_base_init(s_host_t * the_host); 13 | 14 | ///async_threads.cpp (2 matches) 15 | void thread_worker_new(int id); 16 | int thread_list_find_next(); 17 | 18 | ///clnt_main.cpp (8 matches) 19 | void * vnode_download(void * arg); 20 | int node_download(void *); 21 | void * downloadworker(void *); 22 | void clnt_int_handler(const int sig); 23 | void * clnt_int_worker(void * a); 24 | int gingko_clnt_global_init(int argc, char *argv[]); 25 | 26 | ///gingko_common.h (18 matches) 27 | void * helo_serv_s(void *, int); 28 | void * join_job_s(void *, int); 29 | void * quit_job_s(void *, int); 30 | void * dead_host_s(void *, int); 31 | void * get_blocks_s(void *, int); 32 | void * g_none_s(void *, int); 33 | void * new_host_s(void *, int); 34 | void * del_host_s(void *, int); 35 | void * erase_job_s(void *, int); 36 | 37 | ///seed.cpp (4 matches) 38 | int init_struct(const char *name, const struct stat *status, int type, 39 | struct FTW * ftw_info); 40 | int file_counter(const char *name, const struct stat *status, int type, 41 | struct FTW * ftw_info); 42 | void init_total_count_size(s_job_t * jo); 43 | int init_seed(s_job_t * jo); 44 | 45 | ///serv_main.cpp (6 matches) 46 | int init_daemon(void); 47 | inline int pthread_init(); 48 | inline void pthread_clean(); 49 | void serv_int_handler(const int sig); 50 | void * serv_int_worker(void * a); 51 | int gingko_serv_global_init(int argc, char *argv[]); 52 | 53 | ///path.cpp 54 | char * cwd_path_to_abs_path(char * abs_path, const char * oldpath); 55 | 56 | 57 | ///option.cpp 58 | void clnt_show_version(); 59 | void serv_show_version(); 60 | void clnt_show_help(); 61 | void serv_show_help(); 62 | 63 | #endif /* UNITTEST */ 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /testcase/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | killall gingko_serv 4 | cd ccover/output 5 | ./gingko_serv -h 6 | ./gingko_serv --help 7 | ./gingko_serv fdsa fdafda 8 | 9 | 10 | ./gingko_clnt -h 11 | ./gingko_clnt --help 12 | ./gingko_clnt ddd dfa fdafds fda 13 | ./gingko_clnt 10.0.0.0:/home/work/op/oped/noah/tools/gingko/test /home/work/op/oped/noah/tools/gingko/output 14 | ./gingko_serv -u 10 -b 0.0.0.0 -p 2120 -t 10 15 | (./gingko_clnt yf-cm-gingko00.yf01:/home/work/op/oped/noah/tools/gingko/test /home/work/op/oped/noah/tools/gingko/output) 16 | sleep 20 17 | killall gingko_clnt 18 | ./gingko_clnt yf-cm-gingko00.yf01:/home/work/op/oped/noah/tools/gingko/test /home/work/op/oped/noah/tools/gingko/output -c -u 1 -d 20 19 | 20 | -------------------------------------------------------------------------------- /testcase/test.sh.absln: -------------------------------------------------------------------------------- 1 | /home/auxten -------------------------------------------------------------------------------- /testcase/test.sh.ln: -------------------------------------------------------------------------------- 1 | test.sh --------------------------------------------------------------------------------