├── CentOS7 ├── etc │ ├── hosts │ ├── logrotate.d │ │ └── nginx │ ├── nginx │ │ └── nginx.conf │ ├── profile │ └── sysconfig │ │ └── network-scripts │ │ └── ifcfg-ens33 ├── hadoop │ └── hadoop-3.1.3 │ │ └── etc │ │ └── hadoop │ │ ├── core-site.xml │ │ ├── hadoop-env.sh │ │ ├── hdfs-site.xml │ │ ├── mapred-site.xml │ │ └── yarn-site.xml ├── home │ └── user │ │ ├── apps │ │ ├── requirement.txt │ │ ├── templates │ │ │ ├── XSS.html │ │ │ └── test_error.html │ │ └── xss_test_server.py │ │ └── source │ │ └── README └── usr │ └── local │ ├── apache-flume-1.6.0 │ └── conf │ │ ├── flume-log-conf.properties │ │ └── flume.env.sh │ ├── apache-zookeeper-3.6.1 │ ├── conf │ │ ├── zoo1.cfg │ │ ├── zoo2.cfg │ │ └── zoo3.cfg │ └── data │ │ ├── d_1 │ │ └── myid │ │ ├── d_2 │ │ └── myid │ │ └── d_3 │ │ └── myid │ ├── kafka_2.12-2.5.0 │ └── config │ │ └── server-1.properties │ └── spark-2.4.6 │ └── conf │ └── spark-env.sh └── README.md /CentOS7/etc/hosts: -------------------------------------------------------------------------------- 1 | 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 2 | ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 3 | 4 | 192.168.52.130 magic 5 | -------------------------------------------------------------------------------- /CentOS7/etc/logrotate.d/nginx: -------------------------------------------------------------------------------- 1 | /var/log/nginx/*.log { 2 | daily 3 | missingok 4 | rotate 52 5 | compress 6 | delaycompress 7 | notifempty 8 | create 644 nginx adm 9 | sharedscripts 10 | postrotate 11 | if [ -f /var/run/nginx.pid ]; then 12 | kill -USR1 `cat /var/run/nginx.pid` 13 | fi 14 | endscript 15 | } 16 | -------------------------------------------------------------------------------- /CentOS7/etc/nginx/nginx.conf: -------------------------------------------------------------------------------- 1 | 2 | user nginx; 3 | worker_processes 1; 4 | 5 | error_log /var/log/nginx/error.log warn; 6 | pid /var/run/nginx.pid; 7 | 8 | 9 | events { 10 | worker_connections 1024; 11 | } 12 | 13 | 14 | http { 15 | include /etc/nginx/mime.types; 16 | default_type application/octet-stream; 17 | 18 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 19 | '$status $body_bytes_sent "$http_referer" ' 20 | '"$http_user_agent" "$http_x_forwarded_for"'; 21 | 22 | access_log /var/log/nginx/access.log main; 23 | 24 | sendfile on; 25 | #tcp_nopush on; 26 | 27 | keepalive_timeout 65; 28 | 29 | #gzip on; 30 | 31 | server{ 32 | listen 10080; 33 | server_name magic; 34 | location /{ 35 | proxy_pass http://127.0.0.1:12345; 36 | } 37 | } 38 | 39 | 40 | include /etc/nginx/conf.d/*.conf; 41 | } 42 | -------------------------------------------------------------------------------- /CentOS7/etc/profile: -------------------------------------------------------------------------------- 1 | # /etc/profile 2 | 3 | # System wide environment and startup programs, for login setup 4 | # Functions and aliases go in /etc/bashrc 5 | 6 | # It's NOT a good idea to change this file unless you know what you 7 | # are doing. It's much better to create a custom.sh shell script in 8 | # /etc/profile.d/ to make custom changes to your environment, as this 9 | # will prevent the need for merging in future updates. 10 | 11 | pathmunge () { 12 | case ":${PATH}:" in 13 | *:"$1":*) 14 | ;; 15 | *) 16 | if [ "$2" = "after" ] ; then 17 | PATH=$PATH:$1 18 | else 19 | PATH=$1:$PATH 20 | fi 21 | esac 22 | } 23 | 24 | 25 | if [ -x /usr/bin/id ]; then 26 | if [ -z "$EUID" ]; then 27 | # ksh workaround 28 | EUID=`/usr/bin/id -u` 29 | UID=`/usr/bin/id -ru` 30 | fi 31 | USER="`/usr/bin/id -un`" 32 | LOGNAME=$USER 33 | MAIL="/var/spool/mail/$USER" 34 | fi 35 | 36 | # Path manipulation 37 | if [ "$EUID" = "0" ]; then 38 | pathmunge /usr/sbin 39 | pathmunge /usr/local/sbin 40 | else 41 | pathmunge /usr/local/sbin after 42 | pathmunge /usr/sbin after 43 | fi 44 | 45 | HOSTNAME=`/usr/bin/hostname 2>/dev/null` 46 | HISTSIZE=1000 47 | if [ "$HISTCONTROL" = "ignorespace" ] ; then 48 | export HISTCONTROL=ignoreboth 49 | else 50 | export HISTCONTROL=ignoredups 51 | fi 52 | 53 | export PATH USER LOGNAME MAIL HOSTNAME HISTSIZE HISTCONTROL 54 | 55 | # By default, we want umask to get set. This sets it for login shell 56 | # Current threshold for system reserved uid/gids is 200 57 | # You could check uidgid reservation validity in 58 | # /usr/share/doc/setup-*/uidgid file 59 | if [ $UID -gt 199 ] && [ "`/usr/bin/id -gn`" = "`/usr/bin/id -un`" ]; then 60 | umask 002 61 | else 62 | umask 022 63 | fi 64 | 65 | for i in /etc/profile.d/*.sh /etc/profile.d/sh.local ; do 66 | if [ -r "$i" ]; then 67 | if [ "${-#*i}" != "$-" ]; then 68 | . "$i" 69 | else 70 | . "$i" >/dev/null 71 | fi 72 | fi 73 | done 74 | 75 | unset i 76 | unset -f pathmunge 77 | 78 | # Java env 79 | export JAVA_HOME=/usr/local/jdk1.8.0_251 80 | export PATH=$PATH:$JAVA_HOME/bin 81 | 82 | # Hadoop env 83 | export JAVA_HOME=/usr/local/jdk1.8.0_251 84 | export HADOOP_HOME=/hadoop/hadoop-3.1.3 85 | export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/sbin:$HADOOP_HOME/bin 86 | 87 | # zookeeper env 88 | export ZOOKEEPER_HOME=/usr/local/apache-zookeeper-3.6.1 89 | export PATH=$ZOOKEEPER_HOME/bin:$PATH 90 | 91 | # Flume env 92 | export FLUME_HOME=/usr/local/apache-flume-1.6.0 93 | export PATH=$FLUME_HOME/bin:$PATH 94 | 95 | -------------------------------------------------------------------------------- /CentOS7/etc/sysconfig/network-scripts/ifcfg-ens33: -------------------------------------------------------------------------------- 1 | TYPE=Ethernet 2 | PROXY_METHOD=none 3 | BROWSER_ONLY=no 4 | BOOTPROTO=static 5 | DEFROUTE=yes 6 | IPV4_FAILURE_FATAL=no 7 | IPV6INIT=yes 8 | IPV6_AUTOCONF=yes 9 | IPV6_DEFROUTE=yes 10 | IPV6_FAILURE_FATAL=no 11 | IPV6_ADDR_GEN_MODE=stable-privacy 12 | NAME=ens33 13 | UUID=55f92893-e87e-44fd-b89b-d6d813407263 14 | DEVICE=ens33 15 | ONBOOT=yes 16 | IPADDR=192.168.52.130 17 | NETMASK=255.255.255.0 18 | GATEWAY=192.168.52.2 19 | -------------------------------------------------------------------------------- /CentOS7/hadoop/hadoop-3.1.3/etc/hadoop/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | fs.defaultFS 23 | hdfs://magic:9000 24 | 25 | 26 | 27 | hadoop.tmp.dir 28 | /hadoop/data 29 | 30 | 31 | -------------------------------------------------------------------------------- /CentOS7/hadoop/hadoop-3.1.3/etc/hadoop/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | # Set Hadoop-specific environment variables here. 19 | 20 | ## 21 | ## THIS FILE ACTS AS THE MASTER FILE FOR ALL HADOOP PROJECTS. 22 | ## SETTINGS HERE WILL BE READ BY ALL HADOOP COMMANDS. THEREFORE, 23 | ## ONE CAN USE THIS FILE TO SET YARN, HDFS, AND MAPREDUCE 24 | ## CONFIGURATION OPTIONS INSTEAD OF xxx-env.sh. 25 | ## 26 | ## Precedence rules: 27 | ## 28 | ## {yarn-env.sh|hdfs-env.sh} > hadoop-env.sh > hard-coded defaults 29 | ## 30 | ## {YARN_xyz|HDFS_xyz} > HADOOP_xyz > hard-coded defaults 31 | ## 32 | 33 | # Many of the options here are built from the perspective that users 34 | # may want to provide OVERWRITING values on the command line. 35 | # For example: 36 | # 37 | # JAVA_HOME=/usr/java/testing hdfs dfs -ls 38 | # 39 | # Therefore, the vast majority (BUT NOT ALL!) of these defaults 40 | # are configured for substitution and not append. If append 41 | # is preferable, modify this file accordingly. 42 | 43 | ### 44 | # Generic settings for HADOOP 45 | ### 46 | 47 | # Technically, the only required environment variable is JAVA_HOME. 48 | # All others are optional. However, the defaults are probably not 49 | # preferred. Many sites configure these options outside of Hadoop, 50 | # such as in /etc/profile.d 51 | 52 | # The java implementation to use. By default, this environment 53 | # variable is REQUIRED on ALL platforms except OS X! 54 | # export JAVA_HOME= 55 | export JAVA_HOME=/usr/local/jdk1.8.0_251 56 | 57 | # Location of Hadoop. By default, Hadoop will attempt to determine 58 | # this location based upon its execution path. 59 | # export HADOOP_HOME= 60 | 61 | # Location of Hadoop's configuration information. i.e., where this 62 | # file is living. If this is not defined, Hadoop will attempt to 63 | # locate it based upon its execution path. 64 | # 65 | # NOTE: It is recommend that this variable not be set here but in 66 | # /etc/profile.d or equivalent. Some options (such as 67 | # --config) may react strangely otherwise. 68 | # 69 | # export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop 70 | 71 | # The maximum amount of heap to use (Java -Xmx). If no unit 72 | # is provided, it will be converted to MB. Daemons will 73 | # prefer any Xmx setting in their respective _OPT variable. 74 | # There is no default; the JVM will autoscale based upon machine 75 | # memory size. 76 | # export HADOOP_HEAPSIZE_MAX= 77 | 78 | # The minimum amount of heap to use (Java -Xms). If no unit 79 | # is provided, it will be converted to MB. Daemons will 80 | # prefer any Xms setting in their respective _OPT variable. 81 | # There is no default; the JVM will autoscale based upon machine 82 | # memory size. 83 | # export HADOOP_HEAPSIZE_MIN= 84 | 85 | # Enable extra debugging of Hadoop's JAAS binding, used to set up 86 | # Kerberos security. 87 | # export HADOOP_JAAS_DEBUG=true 88 | 89 | # Extra Java runtime options for all Hadoop commands. We don't support 90 | # IPv6 yet/still, so by default the preference is set to IPv4. 91 | # export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true" 92 | # For Kerberos debugging, an extended option set logs more invormation 93 | # export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true -Dsun.security.krb5.debug=true -Dsun.security.spnego.debug" 94 | 95 | # Some parts of the shell code may do special things dependent upon 96 | # the operating system. We have to set this here. See the next 97 | # section as to why.... 98 | export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)} 99 | 100 | # Extra Java runtime options for some Hadoop commands 101 | # and clients (i.e., hdfs dfs -blah). These get appended to HADOOP_OPTS for 102 | # such commands. In most cases, # this should be left empty and 103 | # let users supply it on the command line. 104 | # export HADOOP_CLIENT_OPTS="" 105 | 106 | # 107 | # A note about classpaths. 108 | # 109 | # By default, Apache Hadoop overrides Java's CLASSPATH 110 | # environment variable. It is configured such 111 | # that it sarts out blank with new entries added after passing 112 | # a series of checks (file/dir exists, not already listed aka 113 | # de-deduplication). During de-depulication, wildcards and/or 114 | # directories are *NOT* expanded to keep it simple. Therefore, 115 | # if the computed classpath has two specific mentions of 116 | # awesome-methods-1.0.jar, only the first one added will be seen. 117 | # If two directories are in the classpath that both contain 118 | # awesome-methods-1.0.jar, then Java will pick up both versions. 119 | 120 | # An additional, custom CLASSPATH. Site-wide configs should be 121 | # handled via the shellprofile functionality, utilizing the 122 | # hadoop_add_classpath function for greater control and much 123 | # harder for apps/end-users to accidentally override. 124 | # Similarly, end users should utilize ${HOME}/.hadooprc . 125 | # This variable should ideally only be used as a short-cut, 126 | # interactive way for temporary additions on the command line. 127 | # export HADOOP_CLASSPATH="/some/cool/path/on/your/machine" 128 | 129 | # Should HADOOP_CLASSPATH be first in the official CLASSPATH? 130 | # export HADOOP_USER_CLASSPATH_FIRST="yes" 131 | 132 | # If HADOOP_USE_CLIENT_CLASSLOADER is set, the classpath along 133 | # with the main jar are handled by a separate isolated 134 | # client classloader when 'hadoop jar', 'yarn jar', or 'mapred job' 135 | # is utilized. If it is set, HADOOP_CLASSPATH and 136 | # HADOOP_USER_CLASSPATH_FIRST are ignored. 137 | # export HADOOP_USE_CLIENT_CLASSLOADER=true 138 | 139 | # HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES overrides the default definition of 140 | # system classes for the client classloader when HADOOP_USE_CLIENT_CLASSLOADER 141 | # is enabled. Names ending in '.' (period) are treated as package names, and 142 | # names starting with a '-' are treated as negative matches. For example, 143 | # export HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES="-org.apache.hadoop.UserClass,java.,javax.,org.apache.hadoop." 144 | 145 | # Enable optional, bundled Hadoop features 146 | # This is a comma delimited list. It may NOT be overridden via .hadooprc 147 | # Entries may be added/removed as needed. 148 | # export HADOOP_OPTIONAL_TOOLS="hadoop-kafka,hadoop-aws,hadoop-aliyun,hadoop-openstack,hadoop-azure,hadoop-azure-datalake" 149 | 150 | ### 151 | # Options for remote shell connectivity 152 | ### 153 | 154 | # There are some optional components of hadoop that allow for 155 | # command and control of remote hosts. For example, 156 | # start-dfs.sh will attempt to bring up all NNs, DNS, etc. 157 | 158 | # Options to pass to SSH when one of the "log into a host and 159 | # start/stop daemons" scripts is executed 160 | # export HADOOP_SSH_OPTS="-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10s" 161 | 162 | # The built-in ssh handler will limit itself to 10 simultaneous connections. 163 | # For pdsh users, this sets the fanout size ( -f ) 164 | # Change this to increase/decrease as necessary. 165 | # export HADOOP_SSH_PARALLEL=10 166 | 167 | # Filename which contains all of the hosts for any remote execution 168 | # helper scripts # such as workers.sh, start-dfs.sh, etc. 169 | # export HADOOP_WORKERS="${HADOOP_CONF_DIR}/workers" 170 | 171 | ### 172 | # Options for all daemons 173 | ### 174 | # 175 | 176 | # 177 | # Many options may also be specified as Java properties. It is 178 | # very common, and in many cases, desirable, to hard-set these 179 | # in daemon _OPTS variables. Where applicable, the appropriate 180 | # Java property is also identified. Note that many are re-used 181 | # or set differently in certain contexts (e.g., secure vs 182 | # non-secure) 183 | # 184 | 185 | # Where (primarily) daemon log files are stored. 186 | # ${HADOOP_HOME}/logs by default. 187 | # Java property: hadoop.log.dir 188 | # export HADOOP_LOG_DIR=${HADOOP_HOME}/logs 189 | 190 | # A string representing this instance of hadoop. $USER by default. 191 | # This is used in writing log and pid files, so keep that in mind! 192 | # Java property: hadoop.id.str 193 | # export HADOOP_IDENT_STRING=$USER 194 | 195 | # How many seconds to pause after stopping a daemon 196 | # export HADOOP_STOP_TIMEOUT=5 197 | 198 | # Where pid files are stored. /tmp by default. 199 | # export HADOOP_PID_DIR=/tmp 200 | 201 | # Default log4j setting for interactive commands 202 | # Java property: hadoop.root.logger 203 | # export HADOOP_ROOT_LOGGER=INFO,console 204 | 205 | # Default log4j setting for daemons spawned explicitly by 206 | # --daemon option of hadoop, hdfs, mapred and yarn command. 207 | # Java property: hadoop.root.logger 208 | # export HADOOP_DAEMON_ROOT_LOGGER=INFO,RFA 209 | 210 | # Default log level and output location for security-related messages. 211 | # You will almost certainly want to change this on a per-daemon basis via 212 | # the Java property (i.e., -Dhadoop.security.logger=foo). (Note that the 213 | # defaults for the NN and 2NN override this by default.) 214 | # Java property: hadoop.security.logger 215 | # export HADOOP_SECURITY_LOGGER=INFO,NullAppender 216 | 217 | # Default process priority level 218 | # Note that sub-processes will also run at this level! 219 | # export HADOOP_NICENESS=0 220 | 221 | # Default name for the service level authorization file 222 | # Java property: hadoop.policy.file 223 | # export HADOOP_POLICYFILE="hadoop-policy.xml" 224 | 225 | # 226 | # NOTE: this is not used by default! <----- 227 | # You can define variables right here and then re-use them later on. 228 | # For example, it is common to use the same garbage collection settings 229 | # for all the daemons. So one could define: 230 | # 231 | # export HADOOP_GC_SETTINGS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps" 232 | # 233 | # .. and then use it as per the b option under the namenode. 234 | 235 | ### 236 | # Secure/privileged execution 237 | ### 238 | 239 | # 240 | # Out of the box, Hadoop uses jsvc from Apache Commons to launch daemons 241 | # on privileged ports. This functionality can be replaced by providing 242 | # custom functions. See hadoop-functions.sh for more information. 243 | # 244 | 245 | # The jsvc implementation to use. Jsvc is required to run secure datanodes 246 | # that bind to privileged ports to provide authentication of data transfer 247 | # protocol. Jsvc is not required if SASL is configured for authentication of 248 | # data transfer protocol using non-privileged ports. 249 | # export JSVC_HOME=/usr/bin 250 | 251 | # 252 | # This directory contains pids for secure and privileged processes. 253 | #export HADOOP_SECURE_PID_DIR=${HADOOP_PID_DIR} 254 | 255 | # 256 | # This directory contains the logs for secure and privileged processes. 257 | # Java property: hadoop.log.dir 258 | # export HADOOP_SECURE_LOG=${HADOOP_LOG_DIR} 259 | 260 | # 261 | # When running a secure daemon, the default value of HADOOP_IDENT_STRING 262 | # ends up being a bit bogus. Therefore, by default, the code will 263 | # replace HADOOP_IDENT_STRING with HADOOP_xx_SECURE_USER. If one wants 264 | # to keep HADOOP_IDENT_STRING untouched, then uncomment this line. 265 | # export HADOOP_SECURE_IDENT_PRESERVE="true" 266 | 267 | ### 268 | # NameNode specific parameters 269 | ### 270 | 271 | # Default log level and output location for file system related change 272 | # messages. For non-namenode daemons, the Java property must be set in 273 | # the appropriate _OPTS if one wants something other than INFO,NullAppender 274 | # Java property: hdfs.audit.logger 275 | # export HDFS_AUDIT_LOGGER=INFO,NullAppender 276 | 277 | # Specify the JVM options to be used when starting the NameNode. 278 | # These options will be appended to the options specified as HADOOP_OPTS 279 | # and therefore may override any similar flags set in HADOOP_OPTS 280 | # 281 | # a) Set JMX options 282 | # export HDFS_NAMENODE_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026" 283 | # 284 | # b) Set garbage collection logs 285 | # export HDFS_NAMENODE_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')" 286 | # 287 | # c) ... or set them directly 288 | # export HDFS_NAMENODE_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')" 289 | 290 | # this is the default: 291 | # export HDFS_NAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS" 292 | 293 | ### 294 | # SecondaryNameNode specific parameters 295 | ### 296 | # Specify the JVM options to be used when starting the SecondaryNameNode. 297 | # These options will be appended to the options specified as HADOOP_OPTS 298 | # and therefore may override any similar flags set in HADOOP_OPTS 299 | # 300 | # This is the default: 301 | # export HDFS_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS" 302 | 303 | ### 304 | # DataNode specific parameters 305 | ### 306 | # Specify the JVM options to be used when starting the DataNode. 307 | # These options will be appended to the options specified as HADOOP_OPTS 308 | # and therefore may override any similar flags set in HADOOP_OPTS 309 | # 310 | # This is the default: 311 | # export HDFS_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS" 312 | 313 | # On secure datanodes, user to run the datanode as after dropping privileges. 314 | # This **MUST** be uncommented to enable secure HDFS if using privileged ports 315 | # to provide authentication of data transfer protocol. This **MUST NOT** be 316 | # defined if SASL is configured for authentication of data transfer protocol 317 | # using non-privileged ports. 318 | # This will replace the hadoop.id.str Java property in secure mode. 319 | # export HDFS_DATANODE_SECURE_USER=hdfs 320 | 321 | # Supplemental options for secure datanodes 322 | # By default, Hadoop uses jsvc which needs to know to launch a 323 | # server jvm. 324 | # export HDFS_DATANODE_SECURE_EXTRA_OPTS="-jvm server" 325 | 326 | ### 327 | # NFS3 Gateway specific parameters 328 | ### 329 | # Specify the JVM options to be used when starting the NFS3 Gateway. 330 | # These options will be appended to the options specified as HADOOP_OPTS 331 | # and therefore may override any similar flags set in HADOOP_OPTS 332 | # 333 | # export HDFS_NFS3_OPTS="" 334 | 335 | # Specify the JVM options to be used when starting the Hadoop portmapper. 336 | # These options will be appended to the options specified as HADOOP_OPTS 337 | # and therefore may override any similar flags set in HADOOP_OPTS 338 | # 339 | # export HDFS_PORTMAP_OPTS="-Xmx512m" 340 | 341 | # Supplemental options for priviliged gateways 342 | # By default, Hadoop uses jsvc which needs to know to launch a 343 | # server jvm. 344 | # export HDFS_NFS3_SECURE_EXTRA_OPTS="-jvm server" 345 | 346 | # On privileged gateways, user to run the gateway as after dropping privileges 347 | # This will replace the hadoop.id.str Java property in secure mode. 348 | # export HDFS_NFS3_SECURE_USER=nfsserver 349 | 350 | ### 351 | # ZKFailoverController specific parameters 352 | ### 353 | # Specify the JVM options to be used when starting the ZKFailoverController. 354 | # These options will be appended to the options specified as HADOOP_OPTS 355 | # and therefore may override any similar flags set in HADOOP_OPTS 356 | # 357 | # export HDFS_ZKFC_OPTS="" 358 | 359 | ### 360 | # QuorumJournalNode specific parameters 361 | ### 362 | # Specify the JVM options to be used when starting the QuorumJournalNode. 363 | # These options will be appended to the options specified as HADOOP_OPTS 364 | # and therefore may override any similar flags set in HADOOP_OPTS 365 | # 366 | # export HDFS_JOURNALNODE_OPTS="" 367 | 368 | ### 369 | # HDFS Balancer specific parameters 370 | ### 371 | # Specify the JVM options to be used when starting the HDFS Balancer. 372 | # These options will be appended to the options specified as HADOOP_OPTS 373 | # and therefore may override any similar flags set in HADOOP_OPTS 374 | # 375 | # export HDFS_BALANCER_OPTS="" 376 | 377 | ### 378 | # HDFS Mover specific parameters 379 | ### 380 | # Specify the JVM options to be used when starting the HDFS Mover. 381 | # These options will be appended to the options specified as HADOOP_OPTS 382 | # and therefore may override any similar flags set in HADOOP_OPTS 383 | # 384 | # export HDFS_MOVER_OPTS="" 385 | 386 | ### 387 | # Router-based HDFS Federation specific parameters 388 | # Specify the JVM options to be used when starting the RBF Routers. 389 | # These options will be appended to the options specified as HADOOP_OPTS 390 | # and therefore may override any similar flags set in HADOOP_OPTS 391 | # 392 | # export HDFS_DFSROUTER_OPTS="" 393 | ### 394 | 395 | ### 396 | # Advanced Users Only! 397 | ### 398 | 399 | # 400 | # When building Hadoop, one can add the class paths to the commands 401 | # via this special env var: 402 | # export HADOOP_ENABLE_BUILD_PATHS="true" 403 | 404 | # 405 | # To prevent accidents, shell commands be (superficially) locked 406 | # to only allow certain users to execute certain subcommands. 407 | # It uses the format of (command)_(subcommand)_USER. 408 | # 409 | # For example, to limit who can execute the namenode command, 410 | # export HDFS_NAMENODE_USER=hdfs 411 | -------------------------------------------------------------------------------- /CentOS7/hadoop/hadoop-3.1.3/etc/hadoop/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | dfs.replication 24 | 1 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /CentOS7/hadoop/hadoop-3.1.3/etc/hadoop/mapred-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | mapreduce.framework.name 22 | yarn 23 | 24 | 25 | -------------------------------------------------------------------------------- /CentOS7/hadoop/hadoop-3.1.3/etc/hadoop/yarn-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | yarn.resourcemanager.hostname 23 | magic 24 | 25 | 26 | 27 | yarn.nodemanager.aux-services 28 | mapreduce_shuffle 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /CentOS7/home/user/apps/requirement.txt: -------------------------------------------------------------------------------- 1 | Flask==1.1.2 -------------------------------------------------------------------------------- /CentOS7/home/user/apps/templates/XSS.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | fuckXSS 5 | 43 | 44 | 45 | 46 | 47 |

反射性XSS

48 |
50 | 51 | {% if query %} 52 |
53 | 54 |
55 | anti-XSS 56 |
57 | 58 |
59 | 60 |
Your Search for {{ query|safe }}
61 | Unfortunately it has gone!
62 | 63 | 64 | {% else %} 65 |
66 | 67 |
68 | anti-XSS 69 |
70 | 71 |
72 | {% endif %} 73 | 74 | 75 |
76 | 77 | 78 |

持久性XSS

79 |
81 | 82 | 83 |
84 | 85 |
86 | anti-XSS 87 |
88 | 89 |
90 | 91 |
92 | 93 |

94 |
95 | 96 | 97 | 103 | 104 | {% if comment %} 105 | {% for comment_item in comment %} 106 | 112 | {% endfor %} 113 | {% endif %} 114 | 115 | 116 |
98 |
99 | You
100 | Sun Apr 12 2020 16:44:32 GMT+0800 (中国标准时间) 101 |
Welcome!

This is your personal stream. You can post anything you want here, especially madness.
102 |
107 |
108 | You
109 | 110 |

{{comment_item | safe}}

111 |
117 |
118 | 119 | 120 | 121 | 122 | -------------------------------------------------------------------------------- /CentOS7/home/user/apps/templates/test_error.html: -------------------------------------------------------------------------------- 1 | 2 | 9 |
10 | 11 | 12 | 13 |
-------------------------------------------------------------------------------- /CentOS7/home/user/apps/xss_test_server.py: -------------------------------------------------------------------------------- 1 | # !/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # @time : 2020/04/12 5 | # @Author : Magic 6 | # @File : xss_test_server.py 7 | 8 | 9 | from flask import Flask,render_template,request,redirect,url_for 10 | 11 | app = Flask(__name__) 12 | 13 | 14 | comment = [] 15 | 16 | 17 | @app.route('/', methods=["GET","POST"]) 18 | def index(): 19 | return render_template("XSS.html", comment=comment, query=None) 20 | 21 | 22 | 23 | # Realize the Reflected XSS attack Vulnerable Search box 24 | @app.route('/rxss', methods=["GET","POST"]) 25 | def rxss(): 26 | global qdata 27 | if request.method == "GET": 28 | qdata = request.args.get('query') 29 | action = request.args.get('anti-XSS') 30 | if not action: 31 | action = 0 32 | print(request.args) 33 | if int(action): 34 | qdata = escape(qdata) 35 | print(qdata) 36 | return render_template("XSS.html", query=qdata) 37 | 38 | 39 | # Realize the Persistent XSS attack Vulnerable Comment 40 | @app.route('/pxss', methods=["GET","POST"]) 41 | def pxss(): 42 | if request.method == "POST": 43 | qcomment = request.form.get('qcomment') 44 | action = request.form.get('anti-XSS') 45 | global comment 46 | if not action: 47 | action = 0 48 | print(request.form) 49 | if int(action): 50 | qcomment = escape(qcomment) 51 | comment.append(qcomment) 52 | print(comment) 53 | #return render_template("XSS.html", comment=comment) 54 | return redirect(url_for('index')) 55 | 56 | 57 | # 检查字符串是否含有HTML标签并编码 58 | def escape(s, quote=True): 59 | """ 60 | Replace special characters "&", "<" and ">" to HTML-safe sequences. 61 | If the optional flag quote is true (the default), the quotation mark 62 | characters, both double quote (") and single quote (') characters are also 63 | translated. 64 | """ 65 | s = s.replace("&", "&") # Must be done first! 66 | s = s.replace("<", "<") 67 | s = s.replace(">", ">") 68 | if quote: 69 | s = s.replace('"', """) 70 | s = s.replace('\'', "'") 71 | return s 72 | 73 | 74 | if __name__ == '__main__': 75 | app.run(debug=True, port=12345) 76 | -------------------------------------------------------------------------------- /CentOS7/home/user/source/README: -------------------------------------------------------------------------------- 1 | # Software source 2 | Spark: 3 | version: spark-2.4.6-bin-without-hadoop-scala-2.12 4 | source: https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-2.4.6/spark-2.4.6-bin-without-hadoop-scala-2.12.tgz 5 | 6 | Java8: 7 | version: jdk-8u161-linux-x64 8 | source: https://www.oracle.com/java/technologies/javase/javase-jdk8-downloads.html 9 | 10 | Python3: 11 | version: Python-3.7.0 12 | source: https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tar.xz 13 | 14 | Kafka: 15 | version: kafka_2.12-2.5.0 16 | source: https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/2.5.0/kafka_2.12-2.5.0.tgz 17 | 18 | Hadoop: 19 | version: hadoop-3.1.3 20 | source: https://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-3.1.3/hadoop-3.1.3.tar.gz 21 | 22 | Zookeeper: 23 | version: zookeeper-3.6.1-bin 24 | source: https://mirrors.tuna.tsinghua.edu.cn/apache/zookeeper/zookeeper-3.6.1/apache-zookeeper-3.6.1-bin.tar.gz 25 | 26 | Flume: 27 | version: flume-1.6.0-bin 28 | source: https://mirrors.tuna.tsinghua.edu.cn/apache/flume/1.6.0/apache-flume-1.6.0-bin.tar.gz 29 | -------------------------------------------------------------------------------- /CentOS7/usr/local/apache-flume-1.6.0/conf/flume-log-conf.properties: -------------------------------------------------------------------------------- 1 | agent1.sources=r1 2 | agent1.channels=c1 3 | agent1.sinks=k1 4 | 5 | agent1.sources.r1.type=exec 6 | agent1.sources.r1.command=tail -F /var/log/nginx/access.log 7 | agent1.sources.r1.channels=c1 8 | 9 | agent1.channels.c1.type=memory 10 | agent1.channels.c1.capacity=10000 11 | agent1.channels.c1.transactionCapacity=100 12 | 13 | agent1.sinks.k1.type=org.apache.flume.sink.kafka.KafkaSink 14 | agent1.sinks.k1.topic=log-collect 15 | agent1.sinks.k1.brokerList=192.168.52.130:9092 16 | agent1.sinks.k1.requiredAcks=1 17 | agent1.sinks.k1.batchSize=20 18 | agent1.sinks.k1.channel=c1 19 | -------------------------------------------------------------------------------- /CentOS7/usr/local/apache-flume-1.6.0/conf/flume.env.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # If this file is placed at FLUME_CONF_DIR/flume-env.sh, it will be sourced 18 | # during Flume startup. 19 | 20 | # Enviroment variables can be set here. 21 | 22 | # export JAVA_HOME=/usr/lib/jvm/java-6-sun 23 | export JAVA_HOME=/usr/local/jdk1.8.0_251 24 | 25 | 26 | # Give Flume more memory and pre-allocate, enable remote monitoring via JMX 27 | # export JAVA_OPTS="-Xms100m -Xmx2000m -Dcom.sun.management.jmxremote" 28 | 29 | # Note that the Flume conf directory is always included in the classpath. 30 | #FLUME_CLASSPATH="" 31 | 32 | -------------------------------------------------------------------------------- /CentOS7/usr/local/apache-zookeeper-3.6.1/conf/zoo1.cfg: -------------------------------------------------------------------------------- 1 | # The number of milliseconds of each tick 2 | tickTime=2000 3 | # The number of ticks that the initial 4 | # synchronization phase can take 5 | initLimit=10 6 | # The number of ticks that can pass between 7 | # sending a request and getting an acknowledgement 8 | syncLimit=5 9 | # the directory where the snapshot is stored. 10 | # do not use /tmp for storage, /tmp here is just 11 | # example sakes. 12 | dataDir=/usr/local/apache-zookeeper-3.6.1/data/d_1 13 | # the port at which the clients will connect 14 | clientPort=2181 15 | dataLogDir=/usr/local/apache-zookeeper-3.6.1/logs/log_1 16 | # the maximum number of client connections. 17 | # increase this if you need to handle more clients 18 | #maxClientCnxns=60 19 | # 20 | # Be sure to read the maintenance section of the 21 | # administrator guide before turning on autopurge. 22 | # 23 | # http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance 24 | # 25 | # The number of snapshots to retain in dataDir 26 | #autopurge.snapRetainCount=3 27 | # Purge task interval in hours 28 | # Set to "0" to disable auto purge feature 29 | #autopurge.purgeInterval=1 30 | 31 | ## Metrics Providers 32 | # 33 | # https://prometheus.io Metrics Exporter 34 | #metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider 35 | #metricsProvider.httpPort=7000 36 | #metricsProvider.exportJvmInfo=true 37 | server.1=localhost:2887:3887 38 | server.2=localhost:2888:3888 39 | server.3=localhost:2889:3889 40 | -------------------------------------------------------------------------------- /CentOS7/usr/local/apache-zookeeper-3.6.1/conf/zoo2.cfg: -------------------------------------------------------------------------------- 1 | # The number of milliseconds of each tick 2 | tickTime=2000 3 | # The number of ticks that the initial 4 | # synchronization phase can take 5 | initLimit=10 6 | # The number of ticks that can pass between 7 | # sending a request and getting an acknowledgement 8 | syncLimit=5 9 | # the directory where the snapshot is stored. 10 | # do not use /tmp for storage, /tmp here is just 11 | # example sakes. 12 | 13 | # the port at which the clients will connect 14 | dataDir=/usr/local/apache-zookeeper-3.6.1/data/d_2 15 | clientPort=2182 16 | dataLogDir=/usr/local/apache-zookeeper-3.6.1/logs/log_2 17 | # the maximum number of client connections. 18 | # increase this if you need to handle more clients 19 | #maxClientCnxns=60 20 | # 21 | # Be sure to read the maintenance section of the 22 | # administrator guide before turning on autopurge. 23 | # 24 | # http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance 25 | # 26 | # The number of snapshots to retain in dataDir 27 | #autopurge.snapRetainCount=3 28 | # Purge task interval in hours 29 | # Set to "0" to disable auto purge feature 30 | #autopurge.purgeInterval=1 31 | 32 | ## Metrics Providers 33 | # 34 | # https://prometheus.io Metrics Exporter 35 | #metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider 36 | #metricsProvider.httpPort=7000 37 | #metricsProvider.exportJvmInfo=true 38 | server.1=localhost:2887:3887 39 | server.2=localhost:2888:3888 40 | server.3=localhost:2889:3889 41 | -------------------------------------------------------------------------------- /CentOS7/usr/local/apache-zookeeper-3.6.1/conf/zoo3.cfg: -------------------------------------------------------------------------------- 1 | # The number of milliseconds of each tick 2 | tickTime=2000 3 | # The number of ticks that the initial 4 | # synchronization phase can take 5 | initLimit=10 6 | # The number of ticks that can pass between 7 | # sending a request and getting an acknowledgement 8 | syncLimit=5 9 | # the directory where the snapshot is stored. 10 | # do not use /tmp for storage, /tmp here is just 11 | # example sakes. 12 | 13 | # the port at which the clients will connect 14 | 15 | #modify 16 | dataDir=/usr/local/apache-zookeeper-3.6.1/data/d_3 17 | clientPort=2183 18 | dataLogDir=/usr/local/apache-zookeeper-3.6.1/logs/log_3 19 | server.1=localhost:2887:3887 20 | server.2=localhost:2888:3888 21 | server.3=localhost:2889:3889 22 | # the maximum number of client connections. 23 | # increase this if you need to handle more clients 24 | #maxClientCnxns=60 25 | # 26 | # Be sure to read the maintenance section of the 27 | # administrator guide before turning on autopurge. 28 | # 29 | # http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance 30 | # 31 | # The number of snapshots to retain in dataDir 32 | #autopurge.snapRetainCount=3 33 | # Purge task interval in hours 34 | # Set to "0" to disable auto purge feature 35 | #autopurge.purgeInterval=1 36 | 37 | ## Metrics Providers 38 | # 39 | # https://prometheus.io Metrics Exporter 40 | #metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider 41 | #metricsProvider.httpPort=7000 42 | #metricsProvider.exportJvmInfo=true 43 | 44 | -------------------------------------------------------------------------------- /CentOS7/usr/local/apache-zookeeper-3.6.1/data/d_1/myid: -------------------------------------------------------------------------------- 1 | 1 2 | -------------------------------------------------------------------------------- /CentOS7/usr/local/apache-zookeeper-3.6.1/data/d_2/myid: -------------------------------------------------------------------------------- 1 | 2 2 | -------------------------------------------------------------------------------- /CentOS7/usr/local/apache-zookeeper-3.6.1/data/d_3/myid: -------------------------------------------------------------------------------- 1 | 3 2 | -------------------------------------------------------------------------------- /CentOS7/usr/local/kafka_2.12-2.5.0/config/server-1.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # see kafka.server.KafkaConfig for additional details and defaults 17 | 18 | ############################# Server Basics ############################# 19 | 20 | # The id of the broker. This must be set to a unique integer for each broker. 21 | broker.id=0 22 | 23 | ############################# Socket Server Settings ############################# 24 | 25 | # The address the socket server listens on. It will get the value returned from 26 | # java.net.InetAddress.getCanonicalHostName() if not configured. 27 | # FORMAT: 28 | # listeners = listener_name://host_name:port 29 | # EXAMPLE: 30 | # listeners = PLAINTEXT://your.host.name:9092 31 | listeners=PLAINTEXT://magic:9092 32 | 33 | # Hostname and port the broker will advertise to producers and consumers. If not set, 34 | # it uses the value for "listeners" if configured. Otherwise, it will use the value 35 | # returned from java.net.InetAddress.getCanonicalHostName(). 36 | #advertised.listeners=PLAINTEXT://your.host.name:9092 37 | 38 | # Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details 39 | #listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL 40 | 41 | # The number of threads that the server uses for receiving requests from the network and sending responses to the network 42 | num.network.threads=3 43 | 44 | # The number of threads that the server uses for processing requests, which may include disk I/O 45 | num.io.threads=8 46 | 47 | # The send buffer (SO_SNDBUF) used by the socket server 48 | socket.send.buffer.bytes=102400 49 | 50 | # The receive buffer (SO_RCVBUF) used by the socket server 51 | socket.receive.buffer.bytes=102400 52 | 53 | # The maximum size of a request that the socket server will accept (protection against OOM) 54 | socket.request.max.bytes=104857600 55 | 56 | 57 | ############################# Log Basics ############################# 58 | 59 | # A comma separated list of directories under which to store log files 60 | log.dirs=/home/user/data/kafka/kafkalogs-1 61 | 62 | # The default number of log partitions per topic. More partitions allow greater 63 | # parallelism for consumption, but this will also result in more files across 64 | # the brokers. 65 | num.partitions=1 66 | 67 | # The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. 68 | # This value is recommended to be increased for installations with data dirs located in RAID array. 69 | num.recovery.threads.per.data.dir=1 70 | 71 | ############################# Internal Topic Settings ############################# 72 | # The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" 73 | # For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. 74 | offsets.topic.replication.factor=1 75 | transaction.state.log.replication.factor=1 76 | transaction.state.log.min.isr=1 77 | 78 | ############################# Log Flush Policy ############################# 79 | 80 | # Messages are immediately written to the filesystem but by default we only fsync() to sync 81 | # the OS cache lazily. The following configurations control the flush of data to disk. 82 | # There are a few important trade-offs here: 83 | # 1. Durability: Unflushed data may be lost if you are not using replication. 84 | # 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. 85 | # 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. 86 | # The settings below allow one to configure the flush policy to flush data after a period of time or 87 | # every N messages (or both). This can be done globally and overridden on a per-topic basis. 88 | 89 | # The number of messages to accept before forcing a flush of data to disk 90 | #log.flush.interval.messages=10000 91 | 92 | # The maximum amount of time a message can sit in a log before we force a flush 93 | #log.flush.interval.ms=1000 94 | 95 | ############################# Log Retention Policy ############################# 96 | 97 | # The following configurations control the disposal of log segments. The policy can 98 | # be set to delete segments after a period of time, or after a given size has accumulated. 99 | # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens 100 | # from the end of the log. 101 | 102 | # The minimum age of a log file to be eligible for deletion due to age 103 | log.retention.hours=168 104 | 105 | # A size-based retention policy for logs. Segments are pruned from the log unless the remaining 106 | # segments drop below log.retention.bytes. Functions independently of log.retention.hours. 107 | #log.retention.bytes=1073741824 108 | 109 | # The maximum size of a log segment file. When this size is reached a new log segment will be created. 110 | log.segment.bytes=1073741824 111 | 112 | # The interval at which log segments are checked to see if they can be deleted according 113 | # to the retention policies 114 | log.retention.check.interval.ms=300000 115 | 116 | ############################# Zookeeper ############################# 117 | 118 | # Zookeeper connection string (see zookeeper docs for details). 119 | # This is a comma separated host:port pairs, each corresponding to a zk 120 | # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". 121 | # You can also append an optional chroot string to the urls to specify the 122 | # root directory for all kafka znodes. 123 | zookeeper.connect=magic:2181 124 | 125 | # Timeout in ms for connecting to zookeeper 126 | zookeeper.connection.timeout.ms=18000 127 | 128 | 129 | ############################# Group Coordinator Settings ############################# 130 | 131 | # The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. 132 | # The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. 133 | # The default value for this is 3 seconds. 134 | # We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. 135 | # However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. 136 | group.initial.rebalance.delay.ms=0 137 | -------------------------------------------------------------------------------- /CentOS7/usr/local/spark-2.4.6/conf/spark-env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one or more 5 | # contributor license agreements. See the NOTICE file distributed with 6 | # this work for additional information regarding copyright ownership. 7 | # The ASF licenses this file to You under the Apache License, Version 2.0 8 | # (the "License"); you may not use this file except in compliance with 9 | # the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # 19 | 20 | # This file is sourced when running various Spark programs. 21 | # Copy it as spark-env.sh and edit that to configure Spark for your site. 22 | 23 | # Options read when launching programs locally with 24 | # ./bin/run-example or ./bin/spark-submit 25 | # - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files 26 | # - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node 27 | # - SPARK_PUBLIC_DNS, to set the public dns name of the driver program 28 | 29 | # Options read by executors and drivers running inside the cluster 30 | # - SPARK_LOCAL_IP, to set the IP address Spark binds to on this node 31 | # - SPARK_PUBLIC_DNS, to set the public DNS name of the driver program 32 | # - SPARK_LOCAL_DIRS, storage directories to use on this node for shuffle and RDD data 33 | # - MESOS_NATIVE_JAVA_LIBRARY, to point to your libmesos.so if you use Mesos 34 | 35 | # Options read in YARN client/cluster mode 36 | # - SPARK_CONF_DIR, Alternate conf dir. (Default: ${SPARK_HOME}/conf) 37 | # - HADOOP_CONF_DIR, to point Spark towards Hadoop configuration files 38 | # - YARN_CONF_DIR, to point Spark towards YARN configuration files when you use YARN 39 | # - SPARK_EXECUTOR_CORES, Number of cores for the executors (Default: 1). 40 | # - SPARK_EXECUTOR_MEMORY, Memory per Executor (e.g. 1000M, 2G) (Default: 1G) 41 | # - SPARK_DRIVER_MEMORY, Memory for Driver (e.g. 1000M, 2G) (Default: 1G) 42 | 43 | # Options for the daemons used in the standalone deploy mode 44 | # - SPARK_MASTER_HOST, to bind the master to a different IP address or hostname 45 | # - SPARK_MASTER_PORT / SPARK_MASTER_WEBUI_PORT, to use non-default ports for the master 46 | # - SPARK_MASTER_OPTS, to set config properties only for the master (e.g. "-Dx=y") 47 | # - SPARK_WORKER_CORES, to set the number of cores to use on this machine 48 | # - SPARK_WORKER_MEMORY, to set how much total memory workers have to give executors (e.g. 1000m, 2g) 49 | # - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT, to use non-default ports for the worker 50 | # - SPARK_WORKER_DIR, to set the working directory of worker processes 51 | # - SPARK_WORKER_OPTS, to set config properties only for the worker (e.g. "-Dx=y") 52 | # - SPARK_DAEMON_MEMORY, to allocate to the master, worker and history server themselves (default: 1g). 53 | # - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y") 54 | # - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y") 55 | # - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y") 56 | # - SPARK_DAEMON_CLASSPATH, to set the classpath for all daemons 57 | # - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers 58 | 59 | # Generic options for the daemons used in the standalone deploy mode 60 | # - SPARK_CONF_DIR Alternate conf dir. (Default: ${SPARK_HOME}/conf) 61 | # - SPARK_LOG_DIR Where log files are stored. (Default: ${SPARK_HOME}/logs) 62 | # - SPARK_PID_DIR Where the pid file is stored. (Default: /tmp) 63 | # - SPARK_IDENT_STRING A string representing this instance of spark. (Default: $USER) 64 | # - SPARK_NICENESS The scheduling priority for daemons. (Default: 0) 65 | # - SPARK_NO_DAEMONIZE Run the proposed command in the foreground. It will not output a PID file. 66 | # Options for native BLAS, like Intel MKL, OpenBLAS, and so on. 67 | # You might get better performance to enable these options if using native BLAS (see SPARK-21305). 68 | # - MKL_NUM_THREADS=1 Disable multi-threading of Intel MKL 69 | # - OPENBLAS_NUM_THREADS=1 Disable multi-threading of OpenBLAS 70 | 71 | export SPARK_DIST_CLASSPATH=$(/hadoop/hadoop-3.1.3/bin/hadoop classpath) 72 | 73 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Web-Log-Analyser 2 | WEB访问日志分析与入侵检测可视化系统 3 | 4 | 测试 5 | 6 | Magic test 7 | 8 | Liu test 9 | 10 | lu test 11 | 12 | 13 | 14 | ------ 15 | 16 | update：上传目前所有软件配置至仓库的根文件夹CentOS7中 17 | 18 | --------------------------------------------------------------------------------