Hello,
We recently deployed mongodb on k8s as a replicaset. Everything was working fine until we had to restart the pods.
The arbiter, primary and secondary are up, but they can’t communicate with each other and it’s impossible to connect to the db.
In the logs for the primary we see on startup that the getaddrinfo failed:
I NETWORK [initandlisten] waiting for connections on port 27017
W NETWORK [replexec-0] getaddrinfo(“mongodb-secondary-0.mongodb-headless.mongodb-replicaset.svc.cluster.local”) failed: Name or service not known
D NETWORK [replexec-0] connected to server mongodb-arbiter-0.mongodb-headless.mongodb-replicaset.svc.cluster.local:27017
W NETWORK [replexec-0] getaddrinfo(“mongodb-arbiter-0”) failed: Temporary failure in name resolution
Afterwards there are constant messages on host unreachable:
[Replication] Failed to connect to mongodb-secondary-0.mongodb-headless.mongodb-replicaset.svc.cluster.local:27017 - HostUnreachable: Error connecting to mongodb-secondary-0.mongodb-headless.mongodb-replicaset.svc.cluster.local:27017 :: caused by :: Could not find address for mongodb-secondary-0.mongodb-headless.mongodb-replicaset.svc.cluster.local:27017: SocketException: Host not found (authoritative)
We’re using MONGODB_ADVERTISED_HOSTNAME as such:
- name: MONGODB_ADVERTISED_HOSTNAME
value: >-
$(MONGODB_POD_NAME).mongodb-headless.mongodb-
replicaset.svc.cluster.local
Network in the k8s wasn’t modified, other clusters/namespaces work fine. We tried using a different version of bitnami/mongodb but then reverted to the previous version and configuration which had
worked.
Network endpoints:
apiVersion: v1
kind: Service
metadata:
name: mongodb-headless
namespace: mongodb-replicaset
selfLink: /api/v1/namespaces/mongodb-replicaset/services/mongodb-headless
uid: ea2c62e8-2916-11e9-a6b7-0050568f5646
resourceVersion: '578197287'
creationTimestamp: '2019-02-05T07:23:20Z'
labels:
app: mongodb
chart: mongodb-7.4.4
heritage: Tiller
io.cattle.field/appId: mongodb
release: mongodb
status:
loadBalancer: {}
spec:
ports:
- name: mongodb
protocol: TCP
port: 27017
targetPort: 27017
selector:
app: mongodb
release: mongodb
clusterIP: None
type: ClusterIP
sessionAffinity: None
mongo
apiVersion: v1
kind: Service
metadata:
name: mongodb
namespace: mongodb-replicaset
selfLink: /api/v1/namespaces/mongodb-replicaset/services/mongodb
uid: ea2ff8be-2916-11e9-a6b7-0050568f5646
resourceVersion: '574164609'
creationTimestamp: '2019-02-05T07:23:20Z'
labels:
app: mongodb
chart: mongodb-7.4.4
heritage: Tiller
io.cattle.field/appId: mongodb
release: mongodb
annotations:
field.cattle.io/publicEndpoints: >-
[{"addresses":["10.8.7.22"],"port":27017,"protocol":"TCP","serviceName":"mongodb-replicaset:mongodb","allNodes":true}]
status:
loadBalancer: {}
spec:
ports:
- name: mongodb
protocol: TCP
port: 27017
targetPort: mongodb
nodePort: 27017
selector:
app: mongodb
component: primary
release: mongodb
clusterIP: 10.43.154.112
type: NodePort
sessionAffinity: None
externalTrafficPolicy: Cluster
Pods:
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: mongodb-arbiter
namespace: mongodb-replicaset
selfLink: /apis/apps/v1/namespaces/mongodb-replicaset/statefulsets/mongodb-arbiter
uid: ea31cd4e-2916-11e9-a6b7-0050568f5646
resourceVersion: '578375651'
generation: 146
creationTimestamp: '2019-02-05T07:23:20Z'
labels:
app: mongodb
chart: mongodb-7.4.4
heritage: Tiller
io.cattle.field/appId: mongodb
release: mongodb
status:
observedGeneration: 146
replicas: 1
readyReplicas: 1
currentReplicas: 1
updatedReplicas: 1
currentRevision: mongodb-arbiter-b5bd84ffc
updateRevision: mongodb-arbiter-b5bd84ffc
collisionCount: 0
spec:
replicas: 1
selector:
matchLabels:
app: mongodb
component: arbiter
release: mongodb
template:
metadata:
creationTimestamp: null
labels:
app: mongodb
chart: mongodb-7.4.4
component: arbiter
release: mongodb
spec:
containers:
- name: mongodb-arbiter
image: golem.ilntsur.loc:18080/bitnami/mongodb:4.0.13
ports:
- name: mongodb
containerPort: 27017
protocol: TCP
env:
- name: BITNAMI_DEBUG
value: 'true'
- name: MONGODB_ADVERTISED_HOSTNAME
value: >-
$(MONGODB_POD_NAME).mongodb-headless.mongodb-replicaset.svc.cluster.local
- name: MONGODB_DISABLE_SYSTEM_LOG
value: 'no'
- name: MONGODB_ENABLE_DIRECTORY_PER_DB
value: 'no'
- name: MONGODB_ENABLE_IPV6
value: 'no'
- name: MONGODB_PRIMARY_HOST
value: mongodb
- name: MONGODB_REPLICA_SET_MODE
value: arbiter
- name: MONGODB_REPLICA_SET_NAME
value: mongodb-replicaset
- name: MONGODB_SYSTEM_LOG_VERBOSITY
value: '1'
- name: MONGODB_POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: MONGODB_PRIMARY_ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: mongodb
key: mongodb-root-password
- name: MONGODB_REPLICA_SET_KEY
valueFrom:
secretKeyRef:
name: mongodb
key: mongodb-replica-set-key
resources: {}
livenessProbe:
tcpSocket:
port: mongodb
initialDelaySeconds: 30
timeoutSeconds: 5
periodSeconds: 10
successThreshold: 1
failureThreshold: 6
readinessProbe:
tcpSocket:
port: mongodb
initialDelaySeconds: 5
timeoutSeconds: 5
periodSeconds: 10
successThreshold: 1
failureThreshold: 6
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
imagePullPolicy: IfNotPresent
securityContext:
runAsUser: 1001
runAsNonRoot: true
procMount: Default
restartPolicy: Always
terminationGracePeriodSeconds: 30
dnsPolicy: ClusterFirst
securityContext:
fsGroup: 1001
affinity: {}
schedulerName: default-scheduler
serviceName: mongodb-headless
podManagementPolicy: OrderedReady
updateStrategy:
type: RollingUpdate
rollingUpdate:
partition: 0
revisionHistoryLimit: 10
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: mongodb-primary
namespace: mongodb-replicaset
selfLink: /apis/apps/v1/namespaces/mongodb-replicaset/statefulsets/mongodb-primary
uid: ea3392bf-2916-11e9-a6b7-0050568f5646
resourceVersion: '578202691'
generation: 143
creationTimestamp: '2019-02-05T07:23:20Z'
labels:
app: mongodb
chart: mongodb-7.4.4
heritage: Tiller
io.cattle.field/appId: mongodb
release: mongodb
annotations:
field.cattle.io/publicEndpoints: >-
[{"addresses":["10.8.7.22"],"port":27017,"protocol":"TCP","serviceName":"mongodb-replicaset:mongodb","allNodes":true}]
status:
observedGeneration: 143
replicas: 1
readyReplicas: 1
currentReplicas: 1
updatedReplicas: 1
currentRevision: mongodb-primary-bcd5b684b
updateRevision: mongodb-primary-bcd5b684b
collisionCount: 0
spec:
replicas: 1
selector:
matchLabels:
app: mongodb
component: primary
release: mongodb
template:
metadata:
creationTimestamp: null
labels:
app: mongodb
chart: mongodb-7.4.4
component: primary
release: mongodb
annotations:
cattle.io/timestamp: '2021-07-19T14:05:32Z'
field.cattle.io/ports: >-
[[{"containerPort":27017,"dnsName":"mongodb-primary","kind":"ClusterIP","name":"mongodb","protocol":"TCP"}]]
field.cattle.io/publicEndpoints: >-
[{"addresses":["10.8.7.22"],"allNodes":true,"port":27017,"protocol":"TCP","serviceId":"mongodb-replicaset:mongodb"}]
spec:
containers:
- name: mongodb-primary
image: golem.ilntsur.loc:18080/bitnami/mongodb:4.0.13
ports:
- name: mongodb
containerPort: 27017
protocol: TCP
env:
- name: BITNAMI_DEBUG
value: 'true'
- name: MONGODB_ADVERTISED_HOSTNAME
value: >-
$(MONGODB_POD_NAME).mongodb-headless.mongodb-replicaset.svc.cluster.local
- name: MONGODB_DISABLE_SYSTEM_LOG
value: 'no'
- name: MONGODB_ENABLE_DIRECTORY_PER_DB
value: 'no'
- name: MONGODB_ENABLE_IPV6
value: 'no'
- name: MONGODB_REPLICA_SET_MODE
value: primary
- name: MONGODB_REPLICA_SET_NAME
value: mongodb-replicaset
- name: MONGODB_SYSTEM_LOG_VERBOSITY
value: '0'
- name: MONGODB_POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: MONGODB_ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: mongodb
key: mongodb-root-password
- name: MONGODB_REPLICA_SET_KEY
valueFrom:
secretKeyRef:
name: mongodb
key: mongodb-replica-set-key
resources: {}
volumeMounts:
- name: datadir
mountPath: /bitnami/mongodb
livenessProbe:
exec:
command:
- pgrep
- mongod
initialDelaySeconds: 30
timeoutSeconds: 5
periodSeconds: 10
successThreshold: 1
failureThreshold: 6
readinessProbe:
exec:
command:
- mongo
- '--eval'
- db.adminCommand('ping')
initialDelaySeconds: 5
timeoutSeconds: 5
periodSeconds: 10
successThreshold: 1
failureThreshold: 6
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
imagePullPolicy: IfNotPresent
securityContext:
runAsUser: 1001
runAsNonRoot: true
procMount: Default
restartPolicy: Always
terminationGracePeriodSeconds: 30
dnsPolicy: ClusterFirst
securityContext:
fsGroup: 1001
affinity: {}
schedulerName: default-scheduler
volumeClaimTemplates:
- kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: datadir
creationTimestamp: null
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 8Gi
storageClassName: nfs-client
volumeMode: Filesystem
status:
phase: Pending
serviceName: mongodb-headless
podManagementPolicy: OrderedReady
updateStrategy:
type: RollingUpdate
rollingUpdate:
partition: 0
revisionHistoryLimit: 10
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: mongodb-secondary
namespace: mongodb-replicaset
selfLink: /apis/apps/v1/namespaces/mongodb-replicaset/statefulsets/mongodb-secondary
uid: ea36259c-2916-11e9-a6b7-0050568f5646
resourceVersion: '578203765'
generation: 172
creationTimestamp: '2019-02-05T07:23:20Z'
labels:
app: mongodb
chart: mongodb-7.4.4
heritage: Tiller
io.cattle.field/appId: mongodb
release: mongodb
status:
observedGeneration: 172
replicas: 1
readyReplicas: 1
currentReplicas: 1
updatedReplicas: 1
currentRevision: mongodb-secondary-6f757c5bc
updateRevision: mongodb-secondary-6f757c5bc
collisionCount: 0
spec:
replicas: 1
selector:
matchLabels:
app: mongodb
component: secondary
release: mongodb
template:
metadata:
creationTimestamp: null
labels:
app: mongodb
chart: mongodb-7.4.4
component: secondary
release: mongodb
annotations:
cattle.io/timestamp: '2021-07-19T14:05:28Z'
field.cattle.io/ports: >-
[[{"containerPort":27017,"dnsName":"mongodb-secondary","kind":"ClusterIP","name":"mongodb","protocol":"TCP"}]]
spec:
containers:
- name: mongodb-secondary
image: golem.ilntsur.loc:18080/bitnami/mongodb:4.0.13
ports:
- name: mongodb
containerPort: 27017
protocol: TCP
env:
- name: BITNAMI_DEBUG
value: 'true'
- name: MONGODB_ADVERTISED_HOSTNAME
value: >-
$(MONGODB_POD_NAME).mongodb-headless.mongodb-replicaset.svc.cluster.local
- name: MONGODB_DISABLE_SYSTEM_LOG
value: 'no'
- name: MONGODB_ENABLE_DIRECTORY_PER_DB
value: 'no'
- name: MONGODB_ENABLE_IPV6
value: 'no'
- name: MONGODB_PRIMARY_HOST
value: mongodb
- name: MONGODB_REPLICA_SET_MODE
value: secondary
- name: MONGODB_REPLICA_SET_NAME
value: mongodb-replicaset
- name: MONGODB_SYSTEM_LOG_VERBOSITY
value: '0'
- name: MONGODB_POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: MONGODB_PRIMARY_ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: mongodb
key: mongodb-root-password
- name: MONGODB_REPLICA_SET_KEY
valueFrom:
secretKeyRef:
name: mongodb
key: mongodb-replica-set-key
resources: {}
volumeMounts:
- name: datadir
mountPath: /bitnami/mongodb
livenessProbe:
exec:
command:
- pgrep
- mongod
initialDelaySeconds: 30
timeoutSeconds: 5
periodSeconds: 10
successThreshold: 1
failureThreshold: 6
readinessProbe:
exec:
command:
- mongo
- '--eval'
- db.adminCommand('ping')
initialDelaySeconds: 5
timeoutSeconds: 5
periodSeconds: 10
successThreshold: 1
failureThreshold: 6
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
imagePullPolicy: IfNotPresent
securityContext:
runAsUser: 1001
runAsNonRoot: true
procMount: Default
restartPolicy: Always
terminationGracePeriodSeconds: 30
dnsPolicy: ClusterFirst
securityContext:
fsGroup: 1001
affinity: {}
schedulerName: default-scheduler
volumeClaimTemplates:
- kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: datadir
creationTimestamp: null
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 8Gi
storageClassName: nfs-client
volumeMode: Filesystem
status:
phase: Pending
serviceName: mongodb-headless
podManagementPolicy: Parallel
updateStrategy:
type: RollingUpdate
rollingUpdate:
partition: 0
revisionHistoryLimit: 10
