回复
基于K8s插件版的Jenkins动态节点实践【内含最佳实践】
love374
发布于 2023-7-18 11:35
浏览
0收藏
手动在Jenkins Server上面创建一个Jenkins Agent节点，选择JNLP 类型。然后获取Jenkins Agent与Master连接所需要的JENKINS_AGENT_NAME 和JENKINS_AGENT_SECRET信息。
静态的AGENT节点，需要编写一个Deployment部署文件并传递上面的连接信息，然后kubectl apply；
动态的AGENT节点，需要使用kubernetes插件， 首先配置CLOUD，添加Kubernetes集群的配置信息。然后编写Pipeline Agent部分的YAML和Stages。
1. 创建节点
Jenkins > 节点管理 > 添加节点
配置节点信息
创建成功（后面需要用到节点名称和secret信息）
2. 静态节点
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
k8s-app: jenkinsagent
name: jenkinsagent
namespace: devops
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: jenkinsagent
template:
metadata:
labels:
k8s-app: jenkinsagent
namespace: devops
name: jenkinsagent
spec:
containers:
- name: jenkinsagent
image: jenkins/inbound-agent:4.10-3-jdk8
securityContext:
privileged: true
imagePullPolicy: IfNotPresent
resources:
limits:
cpu: 1000m
memory: 2Gi
requests:
cpu: 500m
memory: 512Mi
env:
- name: JENKINS_URL
value: http://192.168.1.200:8080
- name: JENKINS_SECRET
value: 2f3f27ee602171c5b51bc69c2bda501d703ac65f42de16600b3e3007dfdec422
- name: JENKINS_AGENT_NAME
value: buildserver1
- name: JENKINS_AGENT_WORKDIR
value: /home/jenkins/workspace- 1.
- 2.
- 3.
- 4.
- 5.
- 6.
- 7.
- 8.
- 9.
- 10.
- 11.
- 12.
- 13.
- 14.
- 15.
- 16.
- 17.
- 18.
- 19.
- 20.
- 21.
- 22.
- 23.
- 24.
- 25.
- 26.
- 27.
- 28.
- 29.
- 30.
- 31.
- 32.
- 33.
- 34.
- 35.
- 36.
- 37.
- 38.
- 39.
- 40.
- 41.
- 42.
3. 动态节点
4. 最佳实践
@Library("mylib@feature-k8s") _ //加载共享库
podYaml = """
kind: Pod
apiVersion: v1
metadata:
labels:
k8s-app: jenkinsagent
name: jnlp
namespace: devops
spec:
securityContext:
runAsUser: 0
containers:
- name: jnlp
image: jenkins/inbound-agent:4.10-3-jdk8
imagePullPolicy: IfNotPresent
- name: maven
image: maven:3.8.4-jdk-8
imagePullPolicy: IfNotPresent
command:
- cat
tty: true
- name: dind
image: 'docker:stable-dind'
command:
- dockerd
- --host=unix:///var/run/docker.sock
- --host=tcp://0.0.0.0:8000
- --insecure-registry=192.168.1.200:8088
securityContext:
privileged: true
volumeMounts:
- mountPath: /var/run
name: docker-dir
- name: "docker"
tty: true
image: docker:19.03.15-git
imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: /var/run
name: docker-dir
- name: "curl"
tty: true
image: curlimages/curl:7.84.0
imagePullPolicy: IfNotPresent
command:
- cat
- name: "helm"
tty: true
image: dtzar/helm-kubectl:3.9.0
imagePullPolicy: IfNotPresent
command:
- cat
volumes:
- name: docker-dir
emptyDir: {}
"""
pipeline{
agent{
kubernetes{
label "test01"
cloud 'kubernetes'
yaml podYaml
}
}
options {
skipDefaultCheckout()
}
stages{
stage("Checkout"){
steps{
script {
println("GetCode")
checkout([$class: 'GitSCM',
branches: [[name: "${env.branchName}"]],
extensions: [],
userRemoteConfigs: [[credentialsId: '62a3150f-38f8-4c9d-9334-79072b1d75cc',
url: "${env.srcUrl}"]]])
env.commitID = GetCommitID()
env.buName = "${JOB_NAME}".split("-")[0]
env.serviceName = "${JOB_NAME}".split("_")[0]
currentBuild.description = """ branchName: ${env.branchName} \n"""
currentBuild.displayName = "${env.commitID}"
}
}
}
stage("Build"){
steps{
script{
settingsContent = libraryResource 'config/settings.xml'
writeFile file: 'settings.xml', text: settingsContent
}
container('maven') {
sh "${env.buildShell} -s settings.xml"
}
}
}
stage("PushArtifact"){
steps{
container("curl"){
script{
//Dir /buName/serviceName/version/serviceName-version.xxx
version = "${env.branchName}-${env.commitID}"
// 重命名
JarName = sh returnStdout: true, script: """ls target | grep -E "jar\$" """
fileName = JarName -"\n"
fileType = fileName.split('\\.')[-1]
env.newFileName = "${env.serviceName}-${version}.${fileType}"
sh "cd target ; mv ${fileName} ${newFileName} "
PushArtifact("${env.buName}/${env.serviceName}/${version}", "target", "${newFileName}")
}
}
}
}
stage("Docker"){
steps{
script{
env.registry = "192.168.1.200:8088"
// tag: branch-commid
env.version = "${env.branchName}-${env.commitID}"
env.imageName = "${env.registry}/${env.buName}/${env.serviceName}:${env.version}"
}
container('docker'){
withCredentials([usernamePassword(credentialsId: 'fad4e7c7-1f8b-45b2-83be-2b914bf08edf', passwordVariable: 'DOCKER_PASSWD', usernameVariable: 'DOCKER_USER')]) {
sh """
# 构建镜像
docker build -t ${env.imageName} . --build-arg pkgname=target/${env.newFileName}
#登录镜像仓库
docker login -u ${DOCKER_USER} -p ${DOCKER_PASSWD} ${env.registry}
# 上传镜像
docker push ${env.imageName}
# 删除镜像
sleep 2
docker rmi ${env.imageName}
"""
}
}
}
}
stage("ReleaseFile"){
steps{
container("curl"){
script{
// 下载模板文件
projectId = 18
fileData = GetRepoFile(projectId,"values.yaml", "master" )
//println(fileData)
yamlData = readYaml text: fileData
// 替换模板文件内容
yamlData.image.tag = "${env.version}"
yamlData.image.repository = "${env.registry}/${env.buName}/${env.serviceName}"
sh "rm -fr values.yaml"
writeYaml charset: 'UTF-8', file: 'values.yaml', data: yamlData
sh "cat values.yaml"
//上传替换后的版本文件（新建文件或者更新文件）
newYaml = sh returnStdout: true, script: 'cat values.yaml'
//println(newYaml)
//更新gitlab文件内容
base64Content = newYaml.bytes.encodeBase64().toString()
try {
CreateRepoFile(projectId,"values.yaml",base64Content, "master")
}catch(e){
UpdateRepoFile(projectId,"values.yaml",base64Content, "master")
}
}
}
}
}
}
}
//获取CommitID
def GetCommitID(){
ID = sh returnStdout: true, script:"git rev-parse HEAD"
ID = ID -"\n"
return ID[0..7]
}
//上传制品
def PushArtifact(targetDir, filePath, fileName){
sh """
curl -X POST "http://192.168.1.200:8081/service/rest/v1/components?repository=devops4-local" \
-H "accept: application/json" \
-H "Content-Type: multipart/form-data" \
-F "raw.directory=${targetDir}" \
-F "raw.asset1=@${filePath}/${fileName};type=application/java-archive" \
-F "raw.asset1.filename=${fileName}" \
-u admin:admin123
"""
}
//更新文件内容
def UpdateRepoFile(projectId,filePath,fileContent, branchName){
apiUrl = "projects/${projectId}/repository/files/${filePath}"
reqBody = """{"branch": "${branchName}","encoding":"base64", "content": "${fileContent}", "commit_message": "update a new file"}"""
response = HttpReqByHttpRequest('PUT',apiUrl,reqBody)
println(response)
}
//创建文件
def CreateRepoFile(projectId,filePath,fileContent, branchName){
apiUrl = "projects/${projectId}/repository/files/${filePath}"
reqBody = """{"branch": "${branchName}","encoding":"base64", "content": "${fileContent}", "commit_message": "update a new file"}"""
response = HttpReqByHttpRequest('POST',apiUrl,reqBody)
println(response)
}
//获取文件内容
def GetRepoFile(projectId,filePath, branchName ){
//GET /projects/:id/repository/files/:file_path/raw
apiUrl = "/projects/${projectId}/repository/files/${filePath}/raw?ref=${branchName}"
response = HttpReq('GET', apiUrl)
return response
}
// 封装HTTP
def HttpReqByHttpRequest(reqType, reqUrl,reqBody ){
def gitServer = "http://192.168.1.200:8077/api/v4"
withCredentials([string(credentialsId: '553fcb38-81e1-4cdd-aa60-a8886a2eb323', variable: 'GITLABTOKEN')]) {
response = httpRequest acceptType: 'APPLICATION_JSON_UTF8',
consoleLogResponseBody: true,
contentType: 'APPLICATION_JSON_UTF8',
customHeaders: [[maskValue: false, name: 'PRIVATE-TOKEN', value: "${GITLABTOKEN}"]],
httpMode: "${reqType}",
url: "${gitServer}/${reqUrl}",
wrapAsMultipart: false,
requestBody: "${reqBody}"
}
return response
}
def HttpReq(method, apiUrl){
response = sh returnStdout: true,
script: """
curl --location --request ${method} \
http://192.168.1.200:8077/api/v4/${apiUrl} \
--header 'PRIVATE-TOKEN: N9mvJV4hq-z7yCcYEsC-'
"""
return response
}- 1.
- 2.
- 3.
- 4.
- 5.
- 6.
- 7.
- 8.
- 9.
- 10.
- 11.
- 12.
- 13.
- 14.
- 15.
- 16.
- 17.
- 18.
- 19.
- 20.
- 21.
- 22.
- 23.
- 24.
- 25.
- 26.
- 27.
- 28.
- 29.
- 30.
- 31.
- 32.
- 33.
- 34.
- 35.
- 36.
- 37.
- 38.
- 39.
- 40.
- 41.
- 42.
- 43.
- 44.
- 45.
- 46.
- 47.
- 48.
- 49.
- 50.
- 51.
- 52.
- 53.
- 54.
- 55.
- 56.
- 57.
- 58.
- 59.
- 60.
- 61.
- 62.
- 63.
- 64.
- 65.
- 66.
- 67.
- 68.
- 69.
- 70.
- 71.
- 72.
- 73.
- 74.
- 75.
- 76.
- 77.
- 78.
- 79.
- 80.
- 81.
- 82.
- 83.
- 84.
- 85.
- 86.
- 87.
- 88.
- 89.
- 90.
- 91.
- 92.
- 93.
- 94.
- 95.
- 96.
- 97.
- 98.
- 99.
- 100.
- 101.
- 102.
- 103.
- 104.
- 105.
- 106.
- 107.
- 108.
- 109.
- 110.
- 111.
- 112.
- 113.
- 114.
- 115.
- 116.
- 117.
- 118.
- 119.
- 120.
- 121.
- 122.
- 123.
- 124.
- 125.
- 126.
- 127.
- 128.
- 129.
- 130.
- 131.
- 132.
- 133.
- 134.
- 135.
- 136.
- 137.
- 138.
- 139.
- 140.
- 141.
- 142.
- 143.
- 144.
- 145.
- 146.
- 147.
- 148.
- 149.
- 150.
- 151.
- 152.
- 153.
- 154.
- 155.
- 156.
- 157.
- 158.
- 159.
- 160.
- 161.
- 162.
- 163.
- 164.
- 165.
- 166.
- 167.
- 168.
- 169.
- 170.
- 171.
- 172.
- 173.
- 174.
- 175.
- 176.
- 177.
- 178.
- 179.
- 180.
- 181.
- 182.
- 183.
- 184.
- 185.
- 186.
- 187.
- 188.
- 189.
- 190.
- 191.
- 192.
- 193.
- 194.
- 195.
- 196.
- 197.
- 198.
- 199.
- 200.
- 201.
- 202.
- 203.
- 204.
- 205.
- 206.
- 207.
- 208.
- 209.
- 210.
- 211.
- 212.
- 213.
- 214.
- 215.
- 216.
- 217.
- 218.
- 219.
- 220.
- 221.
- 222.
- 223.
- 224.
- 225.
- 226.
- 227.
- 228.
- 229.
- 230.
- 231.
- 232.
- 233.
- 234.
- 235.
- 236.
- 237.
- 238.
- 239.
- 240.
- 241.
- 242.
- 243.
- 244.
- 245.
- 246.
- 247.
- 248.
- 249.
- 250.
- 251.
- 252.
- 253.
- 254.
- 255.
- 256.
- 257.
- 258.
- 259.
- 260.
- 261.
- 262.
- 263.
- 264.
- 265.
- 266.
- 267.
- 268.
- 269.
- 270.
@Library("mylib@feature-k8s") _ //加载共享库
podYaml = """
kind: Pod
apiVersion: v1
metadata:
labels:
k8s-app: jenkinsagent
name: jnlp
namespace: devops
spec:
securityContext:
runAsUser: 0
containers:
- name: jnlp
image: jenkins/inbound-agent:4.10-3-jdk8
imagePullPolicy: IfNotPresent
- name: maven
image: maven:3.8.4-jdk-8
imagePullPolicy: IfNotPresent
command:
- cat
tty: true
- name: dind
image: 'docker:stable-dind'
command:
- dockerd
- --host=unix:///var/run/docker.sock
- --host=tcp://0.0.0.0:8000
- --insecure-registry=192.168.1.200:8088
securityContext:
privileged: true
volumeMounts:
- mountPath: /var/run
name: docker-dir
- name: "docker"
tty: true
image: docker:19.03.15-git
imagePullPolicy: IfNotPresent
volumeMounts:
- mountPath: /var/run
name: docker-dir
- name: "curl"
tty: true
image: curlimages/curl:7.84.0
imagePullPolicy: IfNotPresent
command:
- cat
- name: "helm"
tty: true
image: dtzar/helm-kubectl:3.9.0
imagePullPolicy: IfNotPresent
command:
- cat
volumes:
- name: docker-dir
emptyDir: {}
"""
env.projectName = "${JOB_NAME}".split("_")[0]
env.buName = "${JOB_NAME}".split("-")[0]
env.serviceName = "${JOB_NAME}".split("_")[0]
pipeline{
agent{
kubernetes{
label "test01"
cloud 'kubernetes'
yaml podYaml
}
}
options {
skipDefaultCheckout()
}
stages{
stage("Checkout"){
steps{
script {
println("GetCode")
sh "[ -d ${env.serviceName} ] || mkdir ${env.serviceName}"
ws("${WORKSPACE}/${env.serviceName}"){
checkout([$class: 'GitSCM',
branches: [[name: "${env.branchName}"]],
extensions: [],
userRemoteConfigs: [[credentialsId: '62a3150f-38f8-4c9d-9334-79072b1d75cc',
url: "${env.srcUrl}"]]])
env.commitID = GetCommitID()
currentBuild.description = """ branchName: ${env.branchName} \n"""
currentBuild.displayName = "${env.commitID}"
}
}
}
}
stage("HelmDeploy"){
steps{
container("helm"){
script{
withCredentials([file(credentialsId: 'cdcb0d0b-7420-4c7a-84dd-a47b57878999', variable: 'KUBE_CONFIG')]) {
content = sh returnStdout: true, script: "cat $KUBE_CONFIG"
writeFile file: './config', text: content
}
sh " mkdir -p /root/.kube ; mv config /root/.kube/"
env.namespace = "${env.buName}"
env.appName = "${env.serviceName}"
currentBuild.description = "NS: ${env.namespace}"
//Helm 发布
sh """
helm package "${env.appName}/"
helm upgrade --install --create-namespace "${env.appName}" ./"${env.appName}"-*.tgz -n ${env.namespace}
helm history "${env.appName}" -n ${env.namespace}
"""
//获取release的历史版本
env.revision = sh returnStdout: true, script: """helm history ${env.appName} -n ${env.namespace} | grep -v 'REVISION' | awk '{print \$1}' """
println("${env.revision}")
println("${env.revision.split('\n').toString()}")
env.REVISION = "${env.revision.split('\n').toString()}"
println("${env.REVISION}")
// 获取应用状态
5.times{
sh "sleep 2; kubectl -n ${env.namespace} get pod | grep ${env.appName}"
}
}
}
}
}
stage("CheckHealth"){
steps{
container("curl"){
script{
sh """
echo "192.168.1.200 devops4.ops.service" >>/etc/hosts
"""
result = sh returnStdout: true, script: """ curl "http://${env.domainName}/health" """
if (result == "ok\n"){
println("success!")
}
}
}
}
}
stage("RollOut"){
input {
message "是否进行回滚"
ok "提交"
submitter "zeyang,aa"
parameters {
choice(choices: ['yes', 'no'], name: 'opts')
}
}
steps{
container("helm"){
script{
switch("${opts}") {
case "yes":
def result = input message: "选择回滚版本?",
parameters: [choice(choices: env.REVISION, name: 'rversion')]
println("${result}")
sh "helm rollback ${env.appName} ${result} -n ${env.namespace} "
break
case "no":
break
}
}
}
}
}
}
}
//获取CommitID
def GetCommitID(){
ID = sh returnStdout: true, script:"git rev-parse HEAD"
ID = ID -"\n"
return ID[0..7]
}- 1.
- 2.
- 3.
- 4.
- 5.
- 6.
- 7.
- 8.
- 9.
- 10.
- 11.
- 12.
- 13.
- 14.
- 15.
- 16.
- 17.
- 18.
- 19.
- 20.
- 21.
- 22.
- 23.
- 24.
- 25.
- 26.
- 27.
- 28.
- 29.
- 30.
- 31.
- 32.
- 33.
- 34.
- 35.
- 36.
- 37.
- 38.
- 39.
- 40.
- 41.
- 42.
- 43.
- 44.
- 45.
- 46.
- 47.
- 48.
- 49.
- 50.
- 51.
- 52.
- 53.
- 54.
- 55.
- 56.
- 57.
- 58.
- 59.
- 60.
- 61.
- 62.
- 63.
- 64.
- 65.
- 66.
- 67.
- 68.
- 69.
- 70.
- 71.
- 72.
- 73.
- 74.
- 75.
- 76.
- 77.
- 78.
- 79.
- 80.
- 81.
- 82.
- 83.
- 84.
- 85.
- 86.
- 87.
- 88.
- 89.
- 90.
- 91.
- 92.
- 93.
- 94.
- 95.
- 96.
- 97.
- 98.
- 99.
- 100.
- 101.
- 102.
- 103.
- 104.
- 105.
- 106.
- 107.
- 108.
- 109.
- 110.
- 111.
- 112.
- 113.
- 114.
- 115.
- 116.
- 117.
- 118.
- 119.
- 120.
- 121.
- 122.
- 123.
- 124.
- 125.
- 126.
- 127.
- 128.
- 129.
- 130.
- 131.
- 132.
- 133.
- 134.
- 135.
- 136.
- 137.
- 138.
- 139.
- 140.
- 141.
- 142.
- 143.
- 144.
- 145.
- 146.
- 147.
- 148.
- 149.
- 150.
- 151.
- 152.
- 153.
- 154.
- 155.
- 156.
- 157.
- 158.
- 159.
- 160.
- 161.
- 162.
- 163.
- 164.
- 165.
- 166.
- 167.
- 168.
- 169.
- 170.
- 171.
- 172.
- 173.
- 174.
- 175.
- 176.
- 177.
- 178.
- 179.
- 180.
- 181.
- 182.
- 183.
- 184.
- 185.
- 186.
- 187.
- 188.
- 189.
- 190.
- 191.
- 192.
- 193.
- 194.
- 195.
- 196.
文章转载自公众号：DevOps云学堂
分类
标签
已于2023-7-18 11:35:48修改
赞
收藏
回复
分享
微博
QQ
微信
举报
举报
微信扫码分享
删除帖子
删除 取消
相关推荐
