docker pull jenkins/jenkins:2.138.2
docker run -p 9090:8080 -p 50000:50000 -v /User/user/jenkins:/var/jenkins_home jenkins

docker run --rm --name jenkins -p 9090:8080 -p 50000:50000 --user root -v /var/run/docker.sock:/var/run/docker.sock -v $(which docker):/usr/bin/docker -v /Users/user/jenkins:/var/jenkins_home jenkins/jenkins:2.138.2

Go

CloudBees Docker Build and Publish:

Go 安装
安装目录：/var/jenkins_home/go

配置访问git证书 SSH Username with private key
harbor jenkins  密码

构建环境
Set up Go programming language tools
构建：
Docker Build and Publish
Docker Host URI 配置 unix:///var/run/docker.sock 或者 tcp://127.0.0.1:2375

pipeline {
    agent { label 'master' }
    tools {
       maven 'maven_1'
    }
     stages {
        stage('Build') {
            steps {
                checkout([$class: 'GitSCM', branches: [[name: '*/master']], 
                doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], 
                userRemoteConfigs: [[url: 'https://github.com/airzhe/demo-junit']]])
                sh 'mvn -version'
                sh 'mvn package -DskipTests'
            }             
        }
        stage('Build docker image') {
            steps {
                //sh 'docker login --username airzhe  --password ×××'
                sh 'docker build -t airzhe/test:${imageversion} .'
                sh 'docker push airzhe/test:${imageversion}'
            }             
        }
    }
    post { 
            failure { 
                echo 'fail !'
            }
            success{
                echo 'success !'
            }
    }
}

<metric name>{<label name>=<label value>, ...}

metric_name [
  "{" label_name "=" `"` label_value `"` { "," label_name "=" `"` label_value `"` } [ "," ] "}"
] value [ timestamp ]

# 不同时间获取不同值，图形上按时间增量展示，如果后面时间戳不写，就使用当前时间，如果获取不到，就为空，图像表示为中间断了如图:   _- -
# HELP sample_http_requests_total The total number of HTTP requests.
# TYPE sample_http_requests_total counter
sample_http_requests_total{method="post",code="200"} 1027 1568018567000
sample_http_requests_total{method="post",code="400"}    3 1568018567000
idelta(sample_http_requests_total[1m]) 获取和一分钟前的差距

#CPU 个数
count(count(node_cpu_seconds_total{instance="172.16.101.209:9100",mode="system"}) by (cpu))
#内存使用率
(1 - (node_memory_MemAvailable_bytes{instance=~"$node"} / (node_memory_MemTotal_bytes{instance=~"$node"})))* 100
#cpu空闲率
avg(rate(node_cpu_seconds_total{mode="idle"}[2m])) by (instance)
#offset25 一分钟前后值差异
delta(sample_http_requests_total{code="200"} [1m] offset 25m ) 

gauge
sum without(device, fstype, mountpoint)(node_filesystem_size_bytes)
max without(device, fstype, mountpoint)(node_filesystem_size_bytes)
avg without(instance, job)(process_open_fds)

counter
#要计算每秒接收的网络流量，可以使用：返回值将是最近5分钟的平均值
rate(node_network_receive_bytes_total[5m])
The output of rate is a gauge, so the same aggregations apply as for gauges.
sum without(device)(rate(node_network_receive_bytes_total[5m]))

//通过rate()函数获取HTTP请求量的增长率
rate(http_requests_total[5m])
//查询当前系统中，访问量前10的HTTP地址
topk(10, http_requests_total)

count without(instance)(process_open_fds > 10)

type就是CPU的不同状态值
idle, nice, user (default), system (default for Windows), iowait, interrupt, softirq, steal
其中idle表示空闲，user表示用户使用

groups:
- name: container-restart
  rules:
  - alert: Containers Restarts (Last 30 Minutes)
    expr: |
      delta(kube_pod_container_status_restarts_total{}[30m])>0  
    for: 5m
    labels:
      severity: warning
      team: DevOps
    annotations:
      summary: "Instance {{ $labels.instance }} down"
      description: "{{$labels.namespace}}/{{$labels.pod}} has many containers restarts in last 30 minutes"

global:
  smtp_smarthost: 'smtp.qq.com:465'
  smtp_from: '532499602@qq.com'
  smtp_auth_username: 'weihaozhe@aa.net'
  smtp_auth_password: ''
  smtp_require_tls: false
route:
  group_by: ['alertname']
  group_wait: 1m
  group_interval: 10m
  repeat_interval: 10m
  receiver: default-receiver
receivers:
- name: 'default-receiver'
  email_configs:
  - to: 'air_zhe@163.com'

https://github.com/jimmidyson/configmap-reload/tree/v0.2.2

LAST.set(time.time())
PromQL表达式time() - hello_world_last_time_seconds 将告诉您自上次请求以来有多少秒。

hello_world_latency_seconds_count是observe已进行的呼叫数，因此rate(hello_world_latency_seconds_count[1m])在表达式浏览器中将返回Hello World请求的每秒速率。

hello_world_latency_seconds_sum是传递给的值的总和 observe，因此rate(hello_world_latency_seconds_sum[1m])每秒响应请求所花费的时间也是如此。

如果将这两个表达式相除，您将获得最后一分钟的平均延迟。 平均延迟的完整表达式为：
rate（hello_world_latency_seconds_sum [1m]）/rate（hello_world_latency_seconds_count [1m]）

sum without()(node_filesystem_size_bytes)
sum by(job, instance, device)(node_filesystem_size_bytes)
sum without(fstype, mountpoint, device)(node_filesystem_size_bytes)
count without(device)(node_disk_read_bytes_total)
avg without(cpu)(rate(node_cpu_seconds_total[5m]))
等于
  sum without(cpu)(rate(node_cpu_seconds_total[5m]))
/
  count without(cpu)(rate(node_cpu_seconds_total[5m]))
max without(device, fstype, mountpoint)(node_filesystem_size_bytes)

topk without(device, fstype, mountpoint)(2, node_filesystem_size_bytes)
分位数
quantile without(cpu)(0.9, rate(node_cpu_seconds_total{mode="system"}[5m]))

kind: Service
apiVersion: v1
metadata:
  annotations:
    prometheus.io/scrape: "true"
    prometheus.io/port: "9121"
  name: redis
  namespace: kube-system

# 两分钟的增长率×60,为什么×60呢？因为为是按秒求的平均值，还原一分钟的就要乘以60，另外prometheus默认1分钟刮一次数据
irate(user_behavior_request_counter[2m])*60

# 统计loki某个job的日志数（通过sum by把不同日期的数据求和，通过count_over_time统计区间向量内每个度量指标的样本数据个数）
sum(count_over_time({job="${job}"} |~"(?i)${search}" [$__interval])) by (job)

# 统计counter类型增长曲线(注意使用变量报警不支持)高版grafana可以使用$__rate_interval, 这个时间和 prometheus 的采集时间设置有关，比如1分钟采集一次，这个值要大于60s
rate(my_test_counter[$__rate_interval])*$__interval_ms/1000

# 统计counter类型一分钟内的增长数
increase(mysql2es_inserted_num[1m])
# 在grafana 中设置`Min interval`为1m，设置Display为Bar

# 按每半小时统计增长数，在grafana使用total计算总数，设置`Min interval`为30m
increase(SOA_SMS_SEND_ANY{attr="message_publish"}[30m]  offset 1d)

该函数会依次对 v 中的每一条时间序列进行处理，通过 regex 匹配 src_label 的值，并将匹配部分 relacement 写入到 dst_label 标签中。如下所示：

label_replace(up, "host", "$1", "instance",  "(.*):.*")
函数处理后，时间序列将包含一个 host 标签，host 标签的值为 Exporter 实例的 IP 地址：

up{host="localhost",instance="localhost:8080",job="cadvisor"}   1
up{host="localhost",instance="localhost:9090",job="prometheus"}   1
up{host="localhost",instance="localhost:9100",job="node"}   1$$

label_replace(BIW_SHT_QUEUE_DELIVERY_ORDER_OUT, "attr", "$1", "attr",  ".*_(.*)")

$kubectl describe node/runner-e480

Normal   NodeHasNoDiskPressure    6m19s (x8 over 6m19s)  kubelet, runner-e480     Node runner-e480 status is now: NodeHasNoDiskPressure

df -h 系统盘使用85%

修改了docker 镜像存储路径
https://blog.csdn.net/glongljl/article/details/80158297

参考:
https://blog.csdn.net/qq_21816375/article/details/82905660

#命令行自动补全
source <(kubectl completion bash)
echo "source <(kubectl completion bash)" >> ~/.bashrc

kubectl get sa --all-namespaces=true
kubectl get roles --all-namespaces=true
kubectl get RoleBinding  --all-namespaces=true
kubectl get secrets --all-namespaces=true
kubectl describe  ClusterRole/cluster-admin


#端口转发 本地2000端口映射到容器3000端口 &……& 目前只能用localhost访问
export POD_NAME=$(kubectl get pods --namespace default -l "app=grafana,release=willing-lamb" -o jsonpath="{.items[0].metadata.name}")
kubectl port-forward willing-lamb-grafana-75d49cb58c-7dn6d 2000:3000


kubectl get secrets -o json | kubectl update -f -

kubectl exec POD_NAME -c CONTAINER_NAME reboot
kubectl exec -it [POD_NAME] -c [CONTAINER_NAME] -- /bin/sh -c "kill 1"

kubectl explain namespace

kubectl get ns default --show-labels
kubectl completion -h

kubectl delete pod deviosow-1828 --namespace=kube-system --grace-period=0 --force

docker images | grep '<none>'| awk '{print $3}' | xargs docker rmi
kubectl -n kube-system get endpoints -o wide

#dns 验证
kubectl run curl --image=radial/busyboxplus:curl -it
nslookup docker-dind-svc.gitlab-managed-apps


#使用命令快速创建 deployment 和 service
kubectl run nginx --image=nginx --replicas=2
kubectl expose deployment nginx --port 80 --external-ip 172.17.8.201

node 上使用 k8s 的core-dns 服务
dig @10.152.183.10 grafana.istio-system.svc.cluster.local


#查看资源yaml
kubectl api-resources | grep pod
kubectl explain podtemplates.template.spec.containers

apiVersion: v1
kind: ServiceAccount
metadata:
  namespace: default
  name: example-sa
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  namespace: default
  name: example-role
rules:
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["get", "watch", "list"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: example-rolebinding
  namespace: default
subjects:
- kind: ServiceAccount
  name: example-sa
  namespace: default
roleRef:
  kind: Role
  name: example-role
  apiGroup: rbac.authorization.k8s.io
  
---
#管理员，角色配置可以参考 kubectl describe  ClusterRole/cluster-admin
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  namespace: default
  name: example-role
rules:
- apiGroups:
  - '*'
  resources:
  - '*'
  verbs:
  - '*'
  
serviceAccountName
serviceAccount #pod请求别的命名空间时的帐号

Error: release community-feature-haozhe-wei failed: namespaces "php-sht" is forbidden: User "system:serviceaccount:gitlab-managed-apps:default" cannot get resource "namespaces" in API group "" in the namespace "php-sht"

更新密钥要小心，因为帐号token会被其他服务关联，比如 tiller account

kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  namespace: gitlab
  name: gitlab-view-role
rules:
- apiGroups:
  - '*'
  resources:
  - '*'
  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: gitlab-view-php-sht-rolebinding
  namespace: gitlab
subjects:
- kind: ServiceAccount
  name: admin
  namespace: php-sht
roleRef:
  kind: Role
  name: gitlab-view-role
  apiGroup: rbac.authorization.k8s.io

admin
maintainer
developer
guest/reporter

每个 pod 都以mount形式挂载这个 默认的Servcice Account，
如：mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"

单独的pod，恢复过程永远发生在当前节点，不会跑到别的节点上去。如果你想让Pod出现在其他的可用节点上，就必须使用 deployment 这样的控制器来管理 pod，哪怕你只需要一个 pod 副本。

可以通过restartPolicy，改变pod的恢复策略

selector 意味着后面这些追加的定义，只会作用于 selector 所定义的，带有"role:frontend"标签的Pod对象

command: ["sh","-c","mkdir /var/www/html ; ln -s /var/www/community/public /var/www/html/public ; nginx -g 'daemon off;'"]

#多行配置
env:
    - name: COMMAND_SCRIPT
      value: |-
        set -xeo pipefail
        helm init --upgrade
        for i in $(seq 1 30); do helm version && break; sleep 1s; echo "Retrying ($i)..."; done
        helm repo add runner https://charts.gitlab.io
        helm repo update
        helm upgrade runner runner/gitlab-runner --install --reset-values --tls --tls-ca-cert /data/helm/runner/config/ca.pem --tls-cert /data/helm/runner/config/cert.pem --tls-key /data/helm/runner/config/key.pem --version 0.4.1 --set rbac.create\=true,rbac.enabled\=true --namespace gitlab-managed-apps -f /data/helm/runner/config/values.yaml


apiVersion: v1
kind: Pod
metadata:
  name: nginx-privileged
spec:
  containers:
    - name: nginx-privileged
      image: nginx:1.14.2
      securityContext:
        privileged: true
        runAsUser: 1000 #指定容器运行账户
      
      

pod 的操作只有创建删除

"hostAliases": [
          {
            "ip": "172.16.101.197",
            "hostnames": [
              "prometheus.local.com"
            ]
          }
]

pod 的标签很很重要，loki用来建立索引，prometheus可以用来指定报警分组.

curl https://192.168.207.237:2376/info --cert ./cert.pm --key ./key.pem  --cacert ./ca.pem
kubectl create secret tls tls-secret --cert=path/to/tls.cert --key=path/to/tls.key

pv 不属于 namespace 
pvc 属于

#标签一定要有 key 可以没有 value
kubectl label/annotate <resource> foo=bar
kubectl label/annotate <resource> foo-

livenessProbe:
  - initialDelaySeconds:5 #容器启动5s后开始执行
    periodSeconds:5 #每5s执行一次

readlinessProbe: #健康检查结果决定这个pod是不是能被通过Service的方式访问到，而并不影响Pod的生命周期

这三种Project Volume 定义的信息，还可以通过环境变量的方式出现在容器里。但环境变量不具备自动更新的能力。所以一般情况下，都建议你好似用 Volume 文件的方式获取这些信息。

projected volume可以映射很多volume源到相同的目录下

#从配置文件生成 configmap
<?php
$c=file_get_contents("conf.php");
echo json_encode($c,JSON_UNESCAPED_UNICODE)."\n";

kubeadm join 172.16.101.197:6443 --token vq0fs8.rzcw1lf6k3lz7986     --discovery-token-ca-cert-hash sha256:68c8228227ae029b091c8d6cdecde4c11ec5dbbbd43fa725060ffdd512fef3cd

节点需要关闭 swap 启动docker服务


还需要下载
k8s.gcr.io/pause:3.1 镜像
k8s.gcr.io/kube-proxy 镜像

在 master节点观察子节点pod创建情况

移除节点:
kubectl delete node node-1

apiVersion: v1
kind: PersistentVolume
metadata:
  name: es-local-pv0
spec:
  capacity:
    storage: 10Gi
  accessModes:
  - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: local-storage
  local:
    path: /data
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - php-cd-node

kubectl run --namespace kube-public redis-client --rm --tty -i --restart='Never' \
 --env REDIS_PASSWORD=$REDIS_PASSWORD \
--image docker.io/bitnami/redis:5.0.5-debian-9-r36 -- bash

apiVersion: v1
kind: Service
metadata:
  name: ldap-chang-password
  namespace: kube-system
spec:
  ports:
  - port: 80
    targetPort: 8080
    protocol: TCP

---

apiVersion: v1
kind: Endpoints
metadata:
  name: ldap-chang-password
  namespace: kube-system
subsets:
  - addresses:
    - ip: 10.111.8.166
    ports:
    - port: 8080
      protocol: TCP

---

apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: uic
  namespace: kube-system
spec:
  rules:
  - host: uic.t1.youhaodongxi.com
    http:
      paths:
      - backend:
          serviceName: ldap-chang-password
          servicePort: 80
        path: /

https://kubernetes.io/docs/tasks/administer-cluster/access-cluster-api/

APISERVER=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')
TOKEN=$(kubectl get secret $(kubectl get serviceaccount default -o jsonpath='{.secrets[0].name}') -o jsonpath='{.data.token}' | base64 --decode )
curl $APISERVER/api --header "Authorization: Bearer $TOKEN" --insecure
{
  "kind": "APIVersions",
  "versions": [
    "v1"
  ],
  "serverAddressByClientCIDRs": [
    {
      "clientCIDR": "0.0.0.0/0",
      "serverAddress": "10.0.1.149:443"
    }
  ]
}

https://stackoverflow.com/questions/47770676/how-to-create-a-kubectl-config-file-for-serviceaccount

https://docs.bitnami.com/kubernetes/how-to/configure-rbac-in-your-kubernetes-cluster/

ports:
  - nodePort: 8482
    port: 8080
    protocol: TCP
    targetPort: 8080

ports:
  - name: http
    nodePort: 8482
    port: 8080
    protocol: TCP
    targetPort: 8080