├── README.md ├── ca ├── centos ├── Dockerfile ├── aliyun-epel.repo └── aliyun-mirror.repo ├── jenkins-pvc.yaml ├── jenkins-rbac.yaml ├── jenkins.yaml ├── ps ├── 0.jpg ├── 1.jpg ├── 2.jpg ├── 3.jpg └── 4.png └── tomcat ├── Dockerfile └── www /README.md: -------------------------------------------------------------------------------- 1 | # Pipeline 介绍 2 | 要实现在 Jenkins 中的构建工作，可以有多种方式，我们这里采用比较常用的 Pipeline 这种方式。Pipeline，简单来说，就是一套运行在 Jenkins 上的工作流框架，将原来独立运行于单个或者多个节点的任务连接起来，实现单个任务难以完成的复杂流程编排和可视化的工作。 3 | 4 | ### 总体架构 5 | 在开始我们的工作之前，让我们花一点时间分析开始使用Jenkins使用Kubernetes容器所需的工作流。Kubernetes对于开发者来说是一个惊人的开源容器编排引擎。Kubernetes是由Google发起的，这使Kubernetes在使用多个开源容器项目方面有一个惊人的优势。默认情况下，Docker更受Kubernetes的使用者支持和青睐。使用Docker容器的工作流程如下图所示： 6 | 7 | ![k8s](./ps/0.jpg) 8 | ### k8s Jenkins构建持续集成流程 9 | ![k8s](./ps/4.png) 10 | ### Docker技术这些应用场景 11 | 参考文档:https://blog.csdn.net/xiegh2014/article/details/80456486 12 | ### 部署准备 13 | ``` 14 | gogs代码管理 15 | git服务器安装参考：https://blog.csdn.net/xiegh2014/article/details/81434421 16 | 17 | Harbor镜像管理 18 | 19 | 持久化存储 20 | NFS安装 21 | 1、使用yum源安装 22 | yum -y install nfs-utils -y 23 | vim /etc/exports 24 | /data/qas *(rw,no_root_squash) 25 | 2、启动nfs服务 26 | 开机启动 27 | systemctl enable rpcbind.service 28 | systemctl enable nfs-server.service 29 | 启动nfs服务 30 | systemctl start rpcbind.service 31 | systemctl start nfs-server.service 32 | 3、检查 NFS 服务器端是否有目录共享 33 | showmount -e 192.168.58.110 34 | 35 | GlusterFS集群安装参考文档：http://blog.51cto.com/passed/2139299 36 | ``` 37 | ### 安装步骤 38 | ``` 39 | 创建namespace命名空间: 40 | kubectl create namespace kube-jenkins 41 | 创建PVC对象： 42 | kubectl create -f jenkins-pvc.yaml 43 | 创建rbac相关的资源对象: 44 | kubectl create -f jenkins-rbac.yaml 45 | 创建Jenkins服务: 46 | kubectl create -f jenkins.yaml 47 | ``` 48 | ### 测试 49 | ``` 50 | node { 51 | stage(' git仓库拉代码') { // for display purposes 52 | sh "echo 'git checkout'" 53 | } 54 | stage('构建代码') { 55 | sh "echo 'Build...'" 56 | } 57 | stage('自动化测试') { 58 | sh "echo 'deploy...'" 59 | } 60 | stage('发布代码K8S') { 61 | sh "echo 'deploy...'" 62 | } 63 | } 64 | ``` 65 | ### 离线包下载 66 | 链接：https://share.weiyun.com/5pCfEWm 密码：kq3dwn 67 | 链接：https://share.weiyun.com/5pf0e0G 密码：pejdye 68 | 链接：https://share.weiyun.com/5m4ADMk 密码：x4uh4c 69 | 链接：https://share.weiyun.com/5FFbM3B 密码：y4fcff 70 | 71 | 72 | openssl pkcs12 -export -out admin.pfx -inkey admin-key.pem -in admin.pem -certfile ca.pem 73 | ### 传统CICD存在的问题 74 | Jenkins 安装完成了，接下来我们不用急着就去使用，我们要了解下在 Kubernetes 环境下面使用 Jenkins 有什么好处。 75 | 76 | 我们知道持续构建与发布是我们日常工作中必不可少的一个步骤，目前大多公司都采用 Jenkins 集群来搭建符合需求的 CI/CD 流程，然而传统的 Jenkins Slave 一主多从方式会存在一些痛点，比如： 77 | 78 | - 主 Master 发生单点故障时，整个流程都不可用了 79 | - 每个 Slave 的配置环境不一样，来完成不同语言的编译打包等操作，但是这些差异化的配置导致管理起来非常不方便，维护起来也是比较费劲 80 | - 资源分配不均衡，有的 Slave 要运行的 job 出现排队等待，而有的 Slave 处于空闲状态 81 | - 资源有浪费，每台 Slave 可能是物理机或者虚拟机，当 Slave 处于空闲状态时，也不会完全释放掉资源。 82 | ### Pipeline&CICD优点 83 | 正因为上面的这些种种痛点，我们渴望一种更高效更可靠的方式来完成这个 CI/CD 流程，而 Docker 虚拟化容器技术能很好的解决这个痛点，又特别是在 Kubernetes 集群环境下面能够更好来解决上面的问题，下图是基于 Kubernetes 搭建 Jenkins 集群的简单示意图： 84 | - 服务高可用，当 Jenkins Master 出现故障时，Kubernetes 会自动创建一个新的 Jenkins Master 容器，并且将 Volume 分配给新创建的容器，保证数据不丢失，从而达到集群服务高可用。 85 | - 动态伸缩，合理使用资源，每次运行 Job 时，会自动创建一个 Jenkins Slave，Job 完成后，Slave 自动注销并删除容器，资源自动释放，而且 Kubernetes 会根据每个资源的使用情况，动态分配 Slave 到空闲的节点上创建，降低出现因某节点资源利用率高，还排队等待在该节点的情况。 86 | - 扩展性好，当 Kubernetes 集群的资源严重不足而导致 Job 排队等待时，可以很容易的添加一个 Kubernetes Node 到集群中，从而实现扩展。 87 | ### 缺点 88 | jnlp-slave pod 无法删除 89 | 如果 jnlp-slave pod创建失败，它会不断的尝试创建新的pod，并试图连接jenkins，一段时间后，就会创造很多失败的jnlp-slave pod。如果遇到这种情况，需 要尽早中断任务并删除失败的pod。（需要手动删除jnlp-slave pod） 90 | ### 沟通交流 91 | 92 | k8s-CICD-Pipeline交流群:18876939 欢迎喜欢交流的朋友加入！ 93 | -------------------------------------------------------------------------------- /ca: -------------------------------------------------------------------------------- 1 | 2 | echo | base64 -d > /opt/ca.crt 3 | echo | base64 -d > /opt/client.crt 4 | echo | base64 -d > /opt/client.key 5 | 6 | openssl pkcs12 -export -out /opt/cert.pfx -inkey /opt/client.key -in /opt/client.crt -certfile /opt/ca.crt 7 | 8 | 9 | git remote rm origin 10 | git remote add origin git@192.168.1.18:mStar/OTT-dual/K3S/supernova 11 | 12 | 链接：https://share.weiyun.com/5aPD4z7 密码：ixuv47 13 | 14 | node { 15 | stage('复制代码') { // for display purposes 16 | sh ''' 17 | sh -x /app/deploy.sh ${HOST} ${DEPLOY_CODE} ${DEPLOY_DIR} 18 | ''' 19 | } 20 | stage('测试') { 21 | sh "echo 'deploy...'" 22 | } 23 | } 24 | http://maven.aliyun.com/nexus/content/groups/public/ 25 | -------------------------------------------------------------------------------- /centos/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:7.4.1708 2 | MAINTAINER Jerry 3 | 4 | ENV TZ "Asia/Shanghai" 5 | ENV TERM xterm 6 | 7 | ADD aliyun-mirror.repo /etc/yum.repos.d/CentOS-Base.repo 8 | ADD aliyun-epel.repo /etc/yum.repos.d/epel.repo 9 | 10 | RUN yum install -y curl wget tar bzip2 unzip vim-enhanced passwd sudo yum-utils hostname net-tools rsync man && \ 11 | yum clean all 12 | -------------------------------------------------------------------------------- /centos/aliyun-epel.repo: -------------------------------------------------------------------------------- 1 | [epel] 2 | name=Extra Packages for Enterprise Linux 7 - $basearch 3 | baseurl=http://mirrors.aliyun.com/epel/7/$basearch 4 | #mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=$basearch 5 | failovermethod=priority 6 | enabled=1 7 | gpgcheck=0 8 | gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7 9 | 10 | [epel-debuginfo] 11 | name=Extra Packages for Enterprise Linux 7 - $basearch - Debug 12 | baseurl=http://mirrors.aliyun.com/epel/7/$basearch/debug 13 | #mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-debug-7&arch=$basearch 14 | failovermethod=priority 15 | enabled=0 16 | gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7 17 | gpgcheck=0 18 | 19 | [epel-source] 20 | name=Extra Packages for Enterprise Linux 7 - $basearch - Source 21 | baseurl=http://mirrors.aliyun.com/epel/7/SRPMS 22 | #mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-source-7&arch=$basearch 23 | failovermethod=priority 24 | enabled=0 25 | gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7 26 | gpgcheck=0 27 | -------------------------------------------------------------------------------- /centos/aliyun-mirror.repo: -------------------------------------------------------------------------------- 1 | # CentOS-Base.repo 2 | # 3 | # The mirror system uses the connecting IP address of the client and the 4 | # update status of each mirror to pick mirrors that are updated to and 5 | # geographically close to the client. You should use this for CentOS updates 6 | # unless you are manually picking other mirrors. 7 | # 8 | # If the mirrorlist= does not work for you, as a fall back you can try the 9 | # remarked out baseurl= line instead. 10 | # 11 | # 12 | 13 | [base] 14 | name=CentOS-$releasever - Base - mirrors.aliyun.com 15 | failovermethod=priority 16 | baseurl=http://mirrors.aliyun.com/centos/$releasever/os/$basearch/ 17 | #mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os 18 | gpgcheck=1 19 | gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7 20 | 21 | #released updates 22 | [updates] 23 | name=CentOS-$releasever - Updates - mirrors.aliyun.com 24 | failovermethod=priority 25 | baseurl=http://mirrors.aliyun.com/centos/$releasever/updates/$basearch/ 26 | #mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=updates 27 | gpgcheck=1 28 | gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7 29 | 30 | #additional packages that may be useful 31 | [extras] 32 | name=CentOS-$releasever - Extras - mirrors.aliyun.com 33 | failovermethod=priority 34 | baseurl=http://mirrors.aliyun.com/centos/$releasever/extras/$basearch/ 35 | #mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=extras 36 | gpgcheck=1 37 | gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7 38 | 39 | #additional packages that extend functionality of existing packages 40 | [centosplus] 41 | name=CentOS-$releasever - Plus - mirrors.aliyun.com 42 | failovermethod=priority 43 | baseurl=http://mirrors.aliyun.com/centos/$releasever/centosplus/$basearch/ 44 | #mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=centosplus 45 | gpgcheck=1 46 | enabled=0 47 | gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7 48 | 49 | #contrib - packages by Centos Users 50 | [contrib] 51 | name=CentOS-$releasever - Contrib - mirrors.aliyun.com 52 | failovermethod=priority 53 | baseurl=http://mirrors.aliyun.com/centos/$releasever/contrib/$basearch/ 54 | #mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=contrib 55 | gpgcheck=1 56 | enabled=0 57 | gpgkey=http://mirrors.aliyun.com/centos/RPM-GPG-KEY-CentOS-7 -------------------------------------------------------------------------------- /jenkins-pvc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolume 3 | metadata: 4 |  name: opspv 5 | spec: 6 |  capacity: 7 |    storage: 20Gi 8 |  accessModes: 9 |  - ReadWriteMany 10 |  persistentVolumeReclaimPolicy: Delete 11 |  nfs: 12 |    server: 172.16.8.48 13 |    path: /data/qas 14 | --- 15 | kind: PersistentVolumeClaim 16 | apiVersion: v1 17 | metadata: 18 |  name: opspvc 19 |  namespace: kube-ops 20 | spec: 21 |  accessModes: 22 |    - ReadWriteMany 23 |  resources: 24 |    requests: 25 |      storage: 20Gi 26 | -------------------------------------------------------------------------------- /jenkins-rbac.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ServiceAccount 3 | metadata: 4 |  name: jenkins 5 |  namespace: kube-ops 6 | --- 7 | kind: Role 8 | apiVersion: rbac.authorization.k8s.io/v1beta1 9 | metadata: 10 |  name: jenkins 11 |  namespace: kube-ops 12 | rules: 13 |  - apiGroups: [""] 14 |    resources: ["pods"] 15 |    verbs: ["create","delete","get","list","patch","update","watch"] 16 |  - apiGroups: [""] 17 |    resources: ["pods/exec"] 18 |    verbs: ["create","delete","get","list","patch","update","watch"] 19 |  - apiGroups: [""] 20 |    resources: ["pods/log"] 21 |    verbs: ["get","list","watch"] 22 |  - apiGroups: [""] 23 |    resources: ["secrets"] 24 |    verbs: ["get"] 25 | --- 26 | apiVersion: rbac.authorization.k8s.io/v1beta1 27 | kind: RoleBinding 28 | metadata: 29 |  name: jenkins 30 |  namespace: kube-ops 31 | roleRef: 32 |  apiGroup: rbac.authorization.k8s.io 33 |  kind: Role 34 |  name: jenkins 35 | subjects: 36 |  - kind: ServiceAccount 37 |    name: jenkins 38 |    namespace: kube-ops 39 | -------------------------------------------------------------------------------- /jenkins.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: extensions/v1beta1 3 | kind: Deployment 4 | metadata: 5 |  name: jenkins 6 |  namespace: kube-ops 7 | spec: 8 |  template: 9 |    metadata: 10 |      labels: 11 |        app: jenkins 12 |    spec: 13 |      terminationGracePeriodSeconds: 10 14 |      containers: 15 |      - name: jenkins 16 |        image: jenkins/jenkins:lts 17 |        imagePullPolicy: IfNotPresent 18 |        ports: 19 |        - containerPort: 8080 20 |          name: web 21 |          protocol: TCP 22 |        - containerPort: 50000 23 |          name: agent 24 |          protocol: TCP 25 |        resources: 26 |          limits: 27 |            cpu: 1000m 28 |            memory: 1Gi 29 |          requests: 30 |            cpu: 500m 31 |            memory: 512Mi 32 |        livenessProbe: 33 |          httpGet: 34 |            path: /login 35 |            port: 8080 36 |          initialDelaySeconds: 60 37 |          timeoutSeconds: 5 38 |          failureThreshold: 12 39 |        readinessProbe: 40 |          httpGet: 41 |            path: /login 42 |            port: 8080 43 |          initialDelaySeconds: 60 44 |          timeoutSeconds: 5 45 |          failureThreshold: 12 46 |        volumeMounts: 47 |        - name: jenkinshome 48 |          subPath: jenkins 49 |          mountPath: /var/jenkins_home 50 |        env: 51 |        - name: LIMITS_MEMORY 52 |          valueFrom: 53 |            resourceFieldRef: 54 |              resource: limits.memory 55 |              divisor: 1Mi 56 |        - name: JAVA_OPTS 57 |          value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85 -Duser.timezone=Asia/Shanghai 58 |      securityContext: 59 |        fsGroup: 1000 60 |      volumes: 61 |      - name: jenkinshome 62 |        persistentVolumeClaim: 63 |          claimName: opspvc 64 | --- 65 | apiVersion: v1 66 | kind: Service 67 | metadata: 68 |  name: jenkins 69 |  namespace: kube-ops 70 |  labels: 71 |    app: jenkins 72 | spec: 73 |  selector: 74 |    app: jenkins 75 |  type: NodePort 76 |  ports: 77 |  - name: web 78 |    port: 8080 79 |    targetPort: web 80 |    nodePort: 38080 81 |  - name: agent 82 |    port: 50000 83 |    targetPort: agent 84 | -------------------------------------------------------------------------------- /ps/0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xgh2016/k8s-CICD-Pipeline/7817c7948919e2a0eafdc1fa1f3aaa7d3a276dac/ps/0.jpg -------------------------------------------------------------------------------- /ps/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xgh2016/k8s-CICD-Pipeline/7817c7948919e2a0eafdc1fa1f3aaa7d3a276dac/ps/1.jpg -------------------------------------------------------------------------------- /ps/2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xgh2016/k8s-CICD-Pipeline/7817c7948919e2a0eafdc1fa1f3aaa7d3a276dac/ps/2.jpg -------------------------------------------------------------------------------- /ps/3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xgh2016/k8s-CICD-Pipeline/7817c7948919e2a0eafdc1fa1f3aaa7d3a276dac/ps/3.jpg -------------------------------------------------------------------------------- /ps/4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xgh2016/k8s-CICD-Pipeline/7817c7948919e2a0eafdc1fa1f3aaa7d3a276dac/ps/4.png -------------------------------------------------------------------------------- /tomcat/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM java:latest 2 | MAINTAINER Jerry 3 | 4 | ADD apache-maven-3.5.3 /usr/local/maven3 5 | ENV MAVEN_HOME /usr/local/maven3 6 | ENV PATH $PATH:$MAVEN_HOME/bin 7 | RUN mkdir -p $MAVEN_HOME 8 | 9 | ADD apache-tomcat-8.5.31 /usr/local/tomcat8 10 | ENV CATALINA_HOME /usr/local/tomcat8 11 | ENV PATH $PATH:$CATALINA_HOME/bin 12 | RUN mkdir -p $CATALINA_HOME 13 | 14 | ENV TZ=Asia/Shanghai 15 | RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone 16 | 17 | WORKDIR $CATALINA_HOME 18 | RUN mvn -version 19 | EXPOSE 8080 20 | 21 | CMD ["/usr/local/tomcat8/bin/catalina.sh", "run"] 22 | -------------------------------------------------------------------------------- /tomcat/www: -------------------------------------------------------------------------------- 1 | node('jenkins-k8s') { 2 | stage('确认回退的版本') { 3 | echo "1.Test Stage" 4 | input "请确认是否测试环境回滚到上一发布版本！" 5 | } 6 | stage('k8s版本回滚') { 7 | echo "2.K8S-Roll-back" 8 | sh ''' 9 | kubectl rollout undo deployment/${Project_Name} -n ${Namespace} 10 | ''' 11 | } 12 | stage('可用性测试') { 13 | echo "3.Test Stage" 14 | } 15 | } 16 | --------------------------------------------------------------------------------