Header Shadow Image


VIRSH: Virtualizing a Physical Rocky 8 Linux Machine

Let's just jump right in into virtualizing a KVM based Physical Server using various KVM tools such as Virsh, Cockpit etc.  Will also introduce a twist by configuring bonding at the end, not the beginning, to document a retrofit to an existing environment. Begin by identifying how the various network interfaces that will make up the setup:

[root@dl380g6-p02 network-scripts]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: enp2s0f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 78:e7:d1:8f:4d:26 brd ff:ff:ff:ff:ff:ff
    inet 10.3.0.10/24 brd 10.3.0.255 scope global noprefixroute enp2s0f0
       valid_lft forever preferred_lft forever
    inet6 fe80::7ae7:d1ff:fe8f:4d26/64 scope link
       valid_lft forever preferred_lft forever
3: enp2s0f1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 78:e7:d1:8f:4d:28 brd ff:ff:ff:ff:ff:ff
4: enp3s0f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 78:e7:d1:8f:4d:2a brd ff:ff:ff:ff:ff:ff
5: enp3s0f1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 78:e7:d1:8f:4d:2c brd ff:ff:ff:ff:ff:ff
[root@dl380g6-p02 network-scripts]#

Begin by installing libvirt (These two commands must be separate for some reason.):

dnf install @virt 
dnf install libvirt-devel virt-top libguestfs-tools wget virt-install virt-viewer

Download Rocky Linux 9 ISO (In this case in /mnt/iso-images folder, after creating it ):

# cd /mnt/iso-images && wget https://download.rockylinux.org/pub/rocky/9/isos/x86_64/Rocky-9.1-x86_64-minimal.iso

Create the directory where thin provisioned drives will exist:

[root@dl380g6-p02 mnt]# mkdir kvm-drives
[root@dl380g6-p02 mnt]# cd kvm-drives/
[root@dl380g6-p02 kvm-drives]# pwd
/mnt/kvm-drives
[root@dl380g6-p02 kvm-drives]#

Enable libvirtd and confirm:

# virsh list –all
# systemctl enable –now libvirtd
# systemctl status libvirtd

Create drives and check them:

# qemu-img info /mnt/kvm-drives/mc-rocky01.nix.mds.xyz-disk01.qcow2
# qemu-img info /mnt/kvm-drives/mc-rocky01.nix.mds.xyz-disk02.qcow2

# qemu-img info /mnt/kvm-drives/mc-rocky01.nix.mds.xyz-disk01.qcow2
image: /mnt/kvm-drives/mc-rocky01.nix.mds.xyz-disk01.qcow2
file format: qcow2
virtual size: 32 GiB (34359738368 bytes)
disk size: 196 KiB
cluster_size: 65536
Format specific information:
    compat: 1.1
    compression type: zlib
    lazy refcounts: false
    refcount bits: 16
    corrupt: false
    extended l2: false
# qemu-img info /mnt/kvm-drives/mc-rocky01.nix.mds.xyz-disk02.qcow2
image: /mnt/kvm-drives/mc-rocky01.nix.mds.xyz-disk02.qcow2
file format: qcow2
virtual size: 64 GiB (68719476736 bytes)
disk size: 196 KiB
cluster_size: 65536
Format specific information:
    compat: 1.1
    compression type: zlib
    lazy refcounts: false
    refcount bits: 16
    corrupt: false
    extended l2: false
[root@dl380g6-p02 kvm-drives]#

Clear all previous definitions in nmcli.  For example:

# nmcli c delete enp2s0f0
# nmcli c delete enp2s0f1
# nmcli c delete enp3s0f0
# nmcli c delete enp3s0f1

# nmcli c delete br0
# nmcli c delete bridge-slave-enp2s0f0

Define bridged networking:

# virsh net-list –all
# nmcli con add ifname br0 type bridge con-name br0 ipv4.addresses 10.3.0.10/24 ipv4.gateway 10.3.0.1 ipv4.dns "192.168.0.46 192.168.0.51 192.168.0.224" ipv4.method manual
# nmcli con add type bridge-slave ifname enp2s0f0 master br0
# nmcli c s

Next, bring the physical interface offline and the bridge interface online.  This is best done via the console since networking will be offline causing you to loose connection.  Check and verify:

# nmcli c down enp2s0f0
# nmcli c up br0
# nmcli c show
# nmcli c show –active
# virsh net-list –all

Next, define the br0 interface in virsh.  Save this content to br0.xml:

<network>
  <name>br0</name>
  <forward mode="bridge"/>
  <bridge name="br0" />
</network>

Next, import the configuration:

# virsh net-define ./br0.xml

Enable autostart and verify:

# virsh net-start br0
# virsh net-autostart br0
# virsh net-list –all

Create a virtual machine:

# virt-install \
–name mc-rocky01.nix.mds.xyz \
–ram 4096 \
–vcpus 4 \
–disk path=/mnt/kvm-drives/mc-rocky01.nix.mds.xyz-disk01.qcow2 \
–disk path=/mnt/kvm-drives/mc-rocky01.nix.mds.xyz-disk02.qcow2 \
–os-variant centos-stream9 \
–os-type linux \
–network bridge=br0,model=virtio \
–graphics vnc,listen=0.0.0.0 \
–console pty,target_type=serial \
–location /mnt/iso-images/Rocky-9.1-x86_64-minimal.iso

NOTE: If the image is in a location that is not accessible, such as /root/, this error will be seen:

ERROR    internal error: process exited while connecting to monitor: 2023-01-23T01:54:21.710369Z qemu-kvm: -blockdev {"driver":"file","filename":"/root/Rocky-9.1-x86_64-minimal.iso","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}: Could not open '/root/Rocky-9.1-x86_64-minimal.iso': Permission denied

ANOTHER NOTE: If using –extra-args='console=ttyS0' to the above virt-install or –nographics, the VNC, SPICE or other graphics options will be skipped and a text based installation will begin.  In this case, the VNC route will be taken though SPICE will also be discussed.

Login to the console and monitor the installation, answering any questions in the process:

# virsh
Welcome to virsh, the virtualization interactive terminal.

Type:  'help' for help with commands
       'quit' to quit

virsh #

virsh # list
 Id   Name                     State
—————————————-
 2    mc-rocky01.nix.mds.xyz   running

virsh # console mc-rocky01.nix.mds.xyz
Connected to domain 'mc-rocky01.nix.mds.xyz'
Escape character is ^] (Ctrl + ])

If there's not activity, look for a message such as this:

WARNING  Unable to connect to graphical console: virt-viewer not installed. Please install the 'virt-viewer' package.
WARNING  No console to launch for the guest, defaulting to –wait -1

Install virt-viewer if not already (see above).  Once installed, find the port on which your new virtual machine is running on:

# netstat -pnltu|grep -Ei qemu
tcp        0      0 0.0.0.0:5900            0.0.0.0:*               LISTEN      7141/qemu-kvm

And verify the machine name from the process table output:

# ps -ef | grep -Ei 7141
qemu        7141       1 19 20:56 ?        00:01:42 /usr/libexec/qemu-kvm -name guest=mc-rocky01.nix.mds.xyz,debug-threads=on -S -object {"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-2-mc-rocky01.nix.mds.x/master-key.aes"} -machine pc-q35-rhel8.6.0,usb=off,dump-guest-core=off,memory-backend=pc.ram -accel kvm -cpu Nehalem-IBRS,vme=on,pdcm=on,x2apic=on,tsc-deadline=on,hypervisor=on,arat=on,tsc-adjust=on,umip=on,stibp=on,arch-capabilities=on,ssbd=on,rdtscp=on,ibpb=on,ibrs=on,amd-stibp=on,amd-ssbd=on,skip-l1dfl-vmentry=on,pschange-mc-no=on -m 4096 -object {"qom-type":"memory-backend-ram","id":"pc.ram","size":4294967296} -overcommit mem-lock=off -smp 4,sockets=4,cores=1,threads=1 -uuid 9ba74db1-1cdf-4940-b108-e9ad9cdb31b5 -no-user-config -nodefaults -chardev socket,id=charmonitor,fd=39,server=on,wait=off -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown -global ICH9-LPC.disable_s3=1 -global ICH9-LPC.disable_s4=1 -boot strict=on -kernel /var/lib/libvirt/boot/virtinst-q_1nkyy8-vmlinuz -initrd /var/lib/libvirt/boot/virtinst-nhyry5q7-initrd.img -device pcie-root-port,port=16,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2 -device pcie-root-port,port=17,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 -device pcie-root-port,port=18,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 -device pcie-root-port,port=19,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 -device pcie-root-port,port=20,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 -device pcie-root-port,port=21,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5 -device pcie-root-port,port=22,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6 -device pcie-root-port,port=23,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7 -device qemu-xhci,p2=15,p3=15,id=usb,bus=pci.2,addr=0x0 -device virtio-serial-pci,id=virtio-serial0,bus=pci.3,addr=0x0 -blockdev {"driver":"file","filename":"/mnt/kvm-drives/mc-rocky01.nix.mds.xyz-disk01.qcow2","node-name":"libvirt-3-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-3-format","read-only":false,"driver":"qcow2","file":"libvirt-3-storage","backing":null} -device virtio-blk-pci,bus=pci.4,addr=0x0,drive=libvirt-3-format,id=virtio-disk0,bootindex=1 -blockdev {"driver":"file","filename":"/mnt/kvm-drives/mc-rocky01.nix.mds.xyz-disk02.qcow2","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-2-format","read-only":false,"driver":"qcow2","file":"libvirt-2-storage","backing":null} -device virtio-blk-pci,bus=pci.5,addr=0x0,drive=libvirt-2-format,id=virtio-disk1 -blockdev {"driver":"file","filename":"/mnt/iso-images/Rocky-9.1-x86_64-minimal.iso","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-1-format","read-only":true,"driver":"raw","file":"libvirt-1-storage"} -device ide-cd,bus=ide.0,drive=libvirt-1-format,id=sata0-0-0 -netdev tap,fd=40,id=hostnet0,vhost=on,vhostfd=42 -device virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:cb:13:ba,bus=pci.1,addr=0x0 -chardev pty,id=charserial0 -device isa-serial,chardev=charserial0,id=serial0 -chardev socket,id=charchannel0,fd=38,server=on,wait=off -device virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0 -device usb-tablet,id=input0,bus=usb.0,port=1 -audiodev {"id":"audio1","driver":"none"} -vnc 0.0.0.0:0,audiodev=audio1 -device VGA,id=video0,vgamem_mb=16,bus=pcie.0,addr=0x1 -device virtio-balloon-pci,id=balloon0,bus=pci.6,addr=0x0 -object {"qom-type":"rng-random","id":"objrng0","filename":"/dev/urandom"} -device virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.7,addr=0x0 -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -msg timestamp=on

# ip a
9: br0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 78:e7:d1:8f:4d:26 brd ff:ff:ff:ff:ff:ff
    inet 10.3.0.10/24 brd 10.3.0.255 scope global noprefixroute br0
       valid_lft forever preferred_lft forever
    inet6 fe80::1af2:4625:48b7:9030/64 scope link noprefixroute
       valid_lft forever preferred_lft forever

 

Next, let's connect to the graphical interface by specifying the IP and above VNC port to view the Graphical install.  Before we do so, we'll need a plugin for our Chrome first:

Chrome Web Store
Home / Apps / Spice Client

Or just search on google.  Once installed, click on launch app to login to a client.  However, this failed for us.  Instead, let's download the Win x64 client instead:

https://virt-manager.org/download/

Look for the Win x64 MSI (gpg) text on the page.  Virt-viewer will get instaleld in something like C:\Program Files\VirtViewer v11.0-256.  Browse to C:\Program Files\VirtViewer v11.0-256\bin folder then start remote-viewer.exe:

https://i2.wp.com/www.microdevsys.com/WordPressImages/virsh-kvm-configuration-virt-viewer-connection-setup.png?ssl=1

However, the above will only work if the graphics specified is spice:

–graphics spice,listen=0.0.0.0

The corresponding element in the xml files is:

<graphics type='spice' autoport='yes' listen='0.0.0.0'>
  <listen type='address' address='0.0.0.0'/>
</graphics>

However, in our case we used VNC.  In this case, the VNC Viewer is required.   That's a different kind of animal:

https://www.realvnc.com/en/connect/download/viewer/

Establish a connection and continue with the Rocky 9 setup:

https://i1.wp.com/www.microdevsys.com/WordPressImages/virsh-kvm-configuration-vnc-viewer-connection-setup.png?ssl=1

https://i2.wp.com/www.microdevsys.com/WordPressImages/virsh-kvm-configuration-vnc-viewer-connection-setup-rocky-install-screen01.png?ssl=1

Continue with the install making the appropriate selections.  Note the dual disk drives specified in the virt-install command.  They're available for our install:

https://i2.wp.com/www.microdevsys.com/WordPressImages/virsh-kvm-configuration-vnc-viewer-connection-setup-rocky-install-drive-selection.png?ssl=1

Suppose Network parameters could have been configured however, point is to test DHCP across the bridge interface br0.  Once installed:

# virt-install \
> –name mc-rocky01.nix.mds.xyz \
> –ram 4096 \
> –vcpus 4 \
> –disk path=/mnt/kvm-drives/mc-rocky01.nix.mds.xyz-disk01.qcow2 \
> –disk path=/mnt/kvm-drives/mc-rocky01.nix.mds.xyz-disk02.qcow2 \
> –os-variant centos-stream9 \
> –os-type linux \
> –network bridge=br0,model=virtio \
> –graphics vnc,listen=0.0.0.0 \
> –console pty,target_type=serial \
> –location /mnt/iso-images/Rocky-9.1-x86_64-minimal.iso
WARNING  Unable to connect to graphical console: virt-viewer not installed. Please install the 'virt-viewer' package.
WARNING  No console to launch for the guest, defaulting to –wait -1

Starting install…
Retrieving file vmlinuz…                                                                                                                          |  11 MB  00:00:00
Retrieving file initrd.img…                                                                                                                       |  88 MB  00:00:00

Domain is still running. Installation may be in progress.
Waiting for the installation to complete.

Domain has shutdown. Continuing.
Domain creation completed.
Restarting guest.

#

verify the IP given once the machine is back up (If prompted for a disk password, since we choose encryption, enter it and proceed with the boot):

https://i2.wp.com/www.microdevsys.com/WordPressImages/virsh-kvm-configuration-guest-dhcp-works.png?ssl=1

Take the time to set the hostname, as per the above image:

# hostnamectl set-hostname mc-rocky01.nix.mds.xyz

Now it's time to test the connectivity from our Windows 10 Laptop:

Using username "root".
root@10.3.0.179's password:
Last login: Sun Jan 22 22:19:46 2023
[root@mc-rocky01 ~]#
[root@mc-rocky01 ~]#
[root@mc-rocky01 ~]#
[root@mc-rocky01 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: enp1s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 52:54:00:cb:13:ba brd ff:ff:ff:ff:ff:ff
    inet 10.3.0.179/24 brd 10.3.0.255 scope global dynamic noprefixroute enp1s0
       valid_lft 3107sec preferred_lft 3107sec
    inet6 fe80::5054:ff:fecb:13ba/64 scope link noprefixroute
       valid_lft forever preferred_lft forever
[root@mc-rocky01 ~]#
[root@mc-rocky01 ~]#
[root@mc-rocky01 ~]# cat /etc/resolv.conf
# Generated by NetworkManager
search nix.mds.xyz mws.mds.xyz mds.xyz
nameserver 192.168.0.46
nameserver 192.168.0.51
nameserver 192.168.0.224
[root@mc-rocky01 ~]#

Note how the DHCP server populated all DNS servers according to the DHCP configuration defined.  Hence external resolution from the KVM guest works and is able to reach out to online sites and resources.  Virsh lists a running machine:

[root@dl380g6-p02 iso-images]# virsh list –all
 Id   Name                     State
—————————————-
 3    mc-rocky01.nix.mds.xyz   running

[root@dl380g6-p02 iso-images]#

and fdisk from the guest KVM machine lists the correct drives:

[root@mc-rocky01 ~]# fdisk -l
Disk /dev/vda: 32 GiB, 34359738368 bytes, 67108864 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disklabel type: dos
Disk identifier: 0xfbc22483

Device     Boot   Start      End  Sectors Size Id Type
/dev/vda1  *       2048  2099199  2097152   1G 83 Linux
/dev/vda2       2099200 67108863 65009664  31G 83 Linux


Disk /dev/vdb: 64 GiB, 68719476736 bytes, 134217728 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes

…………………….

[root@mc-rocky01 ~]

Check block ID's:

[root@mc-rocky01 ~]# blkid /dev/vdb
[root@mc-rocky01 ~]# blkid /dev/vda
/dev/vda: PTUUID="fbc22483" PTTYPE="dos"
[root@mc-rocky01 ~]# blkid /dev/vda1
/dev/vda1: UUID="6d234e64-426d-46d1-a0de-fb4dd0080283" TYPE="xfs" PARTUUID="fbc22483-01"
[root@mc-rocky01 ~]#

 

Time to configure bonding (AKA teaming) to retrofit it into the mix for some HA over the 4 NIC's.  As before, since the network configuration will be adjusted, connectivity will be lost.  It's a good idea to have the console handy: Before doing anything, remove all the configurations (Don't worry about KVM, it will begin to work again once we redefine br01):

# nmcli c
# nmcli c delete br0
# nmcli c delete bridge-slave-enp2s0f0

There should be nothing defined:

# nmcli c
NAME      UUID                                  TYPE                          DEVICE

and the /etc/sysconfig/network-scripts folder should be empty.  Next, let's define the bonding interfaces based on the previous configuration above.  The earlier commands:

# nmcli connection add type bond con-name bond0 ifname bond0 bond.options "mode=active-backup,miimon=100" ipv4.method disabled ipv6.method ignore

OR

# nmcli con add type bond con-name bond0 ifname bond0 mode active-backup ipv4.method disabled ipv6.method ignore ipv4.addresses 10.3.0.10/24 ipv4.gateway 10.3.0.1 ipv4.dns "192.168.0.46 192.168.0.51 192.168.0.224" ipv4.method manual

OR

# nmcli con add type bond con-name bond0 ifname bond0 mode active-backup ipv6.method ignore ipv4.addresses 10.3.0.10/24 ipv4.gateway 10.3.0.1 ipv4.dns "192.168.0.46 192.168.0.51 192.168.0.224" ipv4.method manual

# nmcli con add type bond-slave con-name enp2s0f0 ifname enp2s0f0 master bond0
# nmcli con add type bond-slave con-name enp2s0f1 ifname enp2s0f1 master bond0
# nmcli con add type bond-slave con-name enp3s0f0 ifname enp3s0f0 master bond0
# nmcli con add type bond-slave con-name enp3s0f1 ifname enp3s0f1 master bond0

NOTE:  There is no IP assignment above.  Not needed.  That will go on the br01 interface as before. Activate the connection:

# nmcli c up ifcfg-enp2s0f0
# nmcli c up ifcfg-enp2s0f0
# nmcli c up ifcfg-enp2s0f0
# nmcli c up ifcfg-enp2s0f0

Activate the bond0 interface:

# nmcli con up bond0

Next, reestablish the bridge interface.  IMPORTANT NOTE:  This time the bond0 is added, not the individual physical NIC:

# virsh net-list –all
# nmcli con add ifname br0 type bridge con-name br0 ipv4.addresses 10.3.0.10/24 ipv4.gateway 10.3.0.1 ipv4.dns "192.168.0.46 192.168.0.51 192.168.0.224" ipv4.method manual

Bridges need all interfaces to be added.  NOTE: bond0 of type bridge is incompatible it appears:

# nmcli con add type bridge-slave ifname bond0 master br0
# nmcli con add type bridge-slave ifname enp2s0f0 master br0
# nmcli con add type bridge-slave ifname enp2s0f1 master br0
# nmcli con add type bridge-slave ifname enp3s0f0 master br0
# nmcli con add type bridge-slave ifname enp3s0f1 master br0

# nmcli c up bridge-slave-enp2s0f0
# nmcli c up bridge-slave-enp2s0f1
# nmcli c up bridge-slave-enp3s0f0
# nmcli c up bridge-slave-enp3s0f1

# (optional, not working) nmci c add type vlan con-name vlan0 ifname bond0.0 dev bond0 id 0 master br0 slave-type bridge
# nmcli c s

Test by starting the virtual machine defined earlier:

virsh # start mc-rocky01.nix.mds.xyz
Domain 'mc-rocky01.nix.mds.xyz' started

virsh #

Then ping the physical host:

C:\Users\tom>ping 10.3.0.10 -t

Pinging 10.3.0.10 with 32 bytes of data:
Reply from 10.3.0.10: bytes=32 time=1ms TTL=62
Reply from 10.3.0.10: bytes=32 time=1ms TTL=62

And ping the KVM VM as well:

C:\Users\tom>ping 10.3.0.179 -t

Pinging 10.3.0.179 with 32 bytes of data:
Reply from 10.3.0.179: bytes=32 time=1ms TTL=62
Reply from 10.3.0.179: bytes=32 time=1ms TTL=62

After all is said and done, the interfaces:

[root@dl380g6-p02 ~]# nmcli c
NAME                   UUID                                  TYPE      DEVICE
br0                    a080d0c1-3828-4595-b08f-ed6854354660  bridge    br0
bond0                  ca5a28e1-6bb0-4f43-b2b3-73af73fb877f  bond      bond0
virbr0                 19b49e05-eea8-43ce-be68-a52e50c774b0  bridge    virbr0
vnet0                  05bb4b3f-834a-4a07-b9bf-a3cf22ad5d76  tun       vnet0
bridge-slave-enp2s0f0  7a6300eb-7cbe-4629-97af-a49adc2c15a9  ethernet  enp2s0f0
bridge-slave-enp2s0f1  1eebfee8-b27d-4579-a605-768282c579cd  ethernet  enp2s0f1
bridge-slave-enp3s0f0  a5b96f1f-0d1f-4773-b2fe-243c382aa51b  ethernet  enp3s0f0
bridge-slave-enp3s0f1  a38f12f8-c7af-4047-b024-3d15cadc7eda  ethernet  enp3s0f1

bridge-slave-bond0     97b2404f-a056-452b-a081-d27f7645fb75  ethernet  —
enp2s0f0               750293af-2d1d-4d18-bd75-4b29324afd10  ethernet  —
enp2s0f1               c8261b81-f318-4114-817e-77019d6ff404  ethernet  —
enp3s0f0               09be1a9d-b9f4-4748-8a0a-4793a9426245  ethernet  —
enp3s0f1               b1a8b75e-abec-4488-8f15-d576a4048ed9  ethernet  —

Use the following command to test failover capability:

# ip link set dev enp2s0f0 down
# ip link set dev enp2s0f1 down
# ip link set dev enp3s0f0 down
# ip link set dev enp3s0f1 down

# ip link set dev enp2s0f0 up
# ip link set dev enp2s0f1 up
# ip link set dev enp3s0f0 up
# ip link set dev enp3s0f1 up

As noted above, bond interfaces appear to be incompatible with bridges in Rocky 8+ / RHEL 8+ / CentOS 8+ whereas for RHEL 7 clones, it's sufficient to add the bond0 to br0:

You're now set with bonding and redundancy on the KVM side!

COMING UP!

UI and Cockpit installation (Plus any other goodies I'll think of before completing this post)

Cheers,
TK

REF: RHEL7: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/networking_guide/sec-vlan_on_bond_and_bridge_using_the_networkmanager_command_line_tool_nmcli
REF: RHEL 8: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/8/html/configuring_and_managing_networking/configuring-a-network-bridge_configuring-and-managing-networking#configuring-a-network-bridge-using-nmcli-commands_configuring-a-network-bridge

502 Bad Gateway The server returned an invalid or incomplete response.

Running into this error?

502 Bad Gateway
The server returned an invalid or incomplete response.

This error only popped up when keepalived was started.  Otherwise just with HAproxy, a timeout was seen. It appeared as if it was a keepalived config error.  In this case, it was due to a faulty HAproxy configuration:

[root@jenkins01 jenkins]# diff /etc/haproxy/haproxy.cfg-diff01 /etc/haproxy/haproxy.cfg
46c46
<     server         jenkins01.nix.mds.xyz    jenkins01.nix.mds.xyz:10443 ssl check check-ssl verify none

>     server         jenkins01.nix.mds.xyz    jenkins01.nix.mds.xyz:10443 check check-ssl verify none
[root@jenkins01 jenkins]#

The ssl directive was missing from one of the lines above.

Cheers,
Tom
416 618 8456
 

DHCPD and DHCLIENT: Routes and gateway disappear or go missing or get removed

On one of the Linux hosts, the routes were correctly populated:

# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.0.0.1        0.0.0.0         UG    0      0        0 eth0
10.0.0.0        0.0.0.0         255.255.255.0   U     0      0        0 eth0

however, after a few seconds, the routes were removed:

# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
10.0.0.0        0.0.0.0         255.255.255.0   U     0      0        0 eth0

This then results in:

Request timed out.

or in:

Network is unreachable

The issue was in the DHClient configuration file used by dhclient:

# cat /etc/dhcp/dhclient.conf
supersede dhcp-lease-time 5;
supersede dhcp-rebinding-time 5;
supersede dhcp-renewal-time 5;

This resulted in routes being added and removed too quickly for the software to keep up.  A more appropriate value is below:

# cat /etc/dhcp/dhclient.conf
supersede dhcp-lease-time 86400;
supersede dhcp-rebinding-time 86400;
supersede dhcp-renewal-time 86400;

HTH,
 

ILO: Integrated Lights Out: Accessing Server Management and Console

HP ILO X is the Hewlett Packard server management Integrated Lights Out software running independently from the main circutry of the host itself.  It allows remote management including power on/off, status checks, console etc.  This post goes over how to access the iLO interface when newer browsers such as Chrome, FireFox and Edge don't support the older TLS versions anymore.
 

First, use IETab to login. This works for most scenarios:
https://i2.wp.com/www.microdevsys.com/WordPressImages/ILO2-And-Console-via-IETAB.png?ssl=1

What if that doesn't work?  Or randomly crashes?  Let's try the console means.  Use SSH or PuTTy to login to the iLO interface:

# ssh Administrator@10.0.0.101:22

or PuTTy:

Once logged in, you should see:

Using username "Administrator".
Administrator@192.168.0.42's password:
User:Administrator logged-in to mdsesxi-ilo-p01.mds.xyz(192.168.0.42)
iLO 2 Advanced 2.33 at 14:56:47 Mar 20 2018
Server Name: mdsesxi-ilo-p01.mds.xyz
Server Power: On

</>hpiLO->

Next type help:

</>hpiLO-> help
status=0
status_tag=COMMAND COMPLETED

DMTF SMASH CLP Commands:

help    : Used to get context sensitive help.
show    : Used to show values of a property or contents of a collection target.
create  : Used to create new user account in the name space of the MAP.
 Example: create /map1/accounts1 username=<lname1> password=<pwd12345> name=
 <dname1> group=<admin,config,oemhp_vm,oemhp_rc,oemhp_power>

delete  : Used to delete user account in the name space of the MAP.
 Example: delete /map1/accounts1/<lname1>

load    : Used to move a binary image from an URL to the MAP. The URL is
limited to 80 characters
Example : load /map1/firmware1 -source http://192.168.1.1/images/fw/iLO2_130.bin

reset   : Used to cause a target to cycle from enabled to disabled and back to
      enabled.

set     : Used to set a property or set of properties to a specific value.
start   : Used to cause a target to change state to a higher run level.
stop    : Used to cause a target to change state to a lower run level.
cd      : Used to set the current default target.
 Example: cd targetname

exit    : Used to terminate the CLP session.
version : Used to query the version of the CLP implementation or other CLP
          elements.

oemhp_ping    : Used to determine if an IP address is reachable from this iLO 2.
Example : oemhp_ping /map1 192.168.1.1 , where 192.168.1.1 is the IP address that you wish
          to ping

oemhp_loadSSHKey    : Used to authorize a SSH Key File from an URL The URL is
limited to 80 characters
Example : oemhp_loadSSHKey /map1/config1 -source http://UserName:password@192.168.1.1/images/SSHkey1.ppk

HP CLI Commands:

POWER    : Control server power.
UID      : Control Unit-ID light.
NMI      : Generate an NMI.
VM       : Virtual media commands.
VSP      : Invoke virtual serial port.
VSP LOG  : Invoke virtual serial port data logging.
TEXTCONS : Invoke Remote Text Console on supported platforms.


</>hpiLO->

Note the TEXTCONS command above:

</>hpiLO-> textcons

Starting text console.
Press 'ESC (' to return to the CLI Session.

IMPORTANT: The exit keys are listed above. Some messages will be readable while others will not when host is booting:

Proc 1: Intel(R) Xeon(R) CPU E5540 @ 2.53GHz
Proc 2: Intel(R) Xeon(R) CPU E5540 @ 2.53GHz
QPI Speed: 5.8 GT/s
HP Power Profile Mode: Balanced Power and Performance
Power Regulator Mode: Static Low Power – Processor(s) clocked down to 1.60 GHz

Advanced Memory Protection Mode: Advanced ECC Support
Redundant ROM Detected – This system contains a valid backup system ROM.
Inlet Ambient Temperature: 21C/69F

SATA Option ROM ver 2.00.B12
Copyright 1982, 2008. Hewlett-Packard Development Company, L.P.
  Port1: (CD-ROM) DV-28S-W


Broadcom NetXtreme II Ethernet Boot Agent v6.0.11                 <F9 = Setup>
Copyright (C) 2000-2010 Broadcom Corporation
All rights reserved.
Press Ctrl-S to enter Configuration Menu
Integrated Lights-Out 2 Advanced
iLO 2 v2.33 Mar 20 2018 10.3.0.8

Slot 0  HP Smart Array P410i Controller      Initializing…  \

If the message below is seen:

Monitor is in graphics mode or an unsupported text mode.

Add in the following kernel boot options:

vga=normal nomodeset

to:

# cat /etc/default/grub
GRUB_TIMEOUT=5
GRUB_DEFAULT=saved
GRUB_DISABLE_SUBMENU=true
GRUB_TERMINAL_OUTPUT="console"
GRUB_CMDLINE_LINUX="rd.lvm.lv=rhel/swap crashkernel=auto rd.lvm.lv=rhel/root rhgb quiet net.ifnames=0 vga=normal nomodeset"
GRUB_DISABLE_RECOVERY="true"

Then stage the update:

# grub2-mkconfig –o /boot/grub2/grub.cfg

and reboot that Linux instance, then try again via textconsole:

Probing EDD (edd=off to disable)… ok                                      
Rocky Linux 8.5 (Green Obsidian)                                            
Kernel 4.18.0-348.el8.0.2.x86_64 on an x86_64                                                                                                          
Activate the web console with: systemctl enable –now cockpit.socket                                                                                    
dl380g6-p02 login:    
   

GL!

HTH,
TK

REF: https://docs.rockylinux.org/books/admin_guide/10-boot/

 

ipa-client-install missing reverse records

For each reverse zone, when manually created, such as:

DNS Zone: 0.168.192.in-addr.arpa.

or

DNS Zone: 0.0.10.in-addr.arpa.

for example, in order for FreeIPA to create reverse records, the option:

Dynamic Update

must be set to True in the reverse zone Settings tab.  If not enabled, messages such as these will be seen when installing clients using ipa-client-install:

Hostname (lumberjack01.unix.my.dom) does not have A/AAAA record.

Cheers,
TK

DD-WRT: Network Performance Tuning

It became apparent that with the growing push for more content on web pages and general media content, my router began to perform rather inadequately.  Inadequately to the point where it was rebooting spontaneously.  Below is what the situation looked like.  Below is an example of high SIRQ's inundating the environment: 

Mem: 116784K used, 396932K free, 432K shrd, 84K buff, 13164K cached
CPU0:  0.0% usr  0.0% sys  0.0% nic  0.0% idle  0.0% io  0.0% irq  100% sirq
CPU1:  0.0% usr  2.5% sys  0.0% nic 96.0% idle  0.0% io  0.0% irq  1.2% sirq
Load average: 3.53 1.16 0.49 4/112 9210
  PID  PPID USER     STAT   VSZ %VSZ CPU %CPU COMMAND
 1068     1 root     R     1496  0.2   0 35.9 syslogd -Z -L -R 192.168.0.14
 2866     2 root     SW       0  0.0   1  0.6 [kworker/1:0]
 6865     1 root     S     1524  0.3   1  0.2 watchquagga -dz -r %s -d zebra ospfd bgpd ripd
  206     2 root     SW       0  0.0   1  0.1 [kswapd0]
 8677  2731 root     R     1520  0.3   1  0.0 top -d 10

Mem: 113748K used, 399968K free, 288K shrd, 8084K buff, 13796K cached
CPU0:  0.0% usr  0.0% sys  0.0% nic  0.0% idle  0.0% io  0.0% irq  100% sirq
CPU1:  0.0% usr  0.0% sys  0.0% nic 83.3% idle  0.0% io  0.0% irq 16.6% sirq
Load average: 0.92 0.60 0.24 3/115 2585
  PID  PPID USER     STAT   VSZ %VSZ CPU %CPU COMMAND
    3     2 root     RW       0  0.0   0 40.9 [ksoftirqd/0]
  735     1 root     D     1624  0.3   1  3.1 watchdog
 2585  2580 root     R     1512  0.2   1  0.1 top -d 10
  721     2 root     RW       0  0.0   1  0.1 [dhd_watchdog_th]

Also the Destination Host Unreachable due to the load:

[1668315026.612617] 64 bytes from 192.168.0.16: icmp_seq=2996 ttl=64 time=14.4 ms
[1668315027.996485] 64 bytes from 192.168.0.16: icmp_seq=2997 ttl=64 time=396 ms
[1668315057.607704] From 192.168.0.114 icmp_seq=3024 Destination Host Unreachable
[1668315057.607796] From 192.168.0.114 icmp_seq=3025 Destination Host Unreachable
[1668315057.607808] From 192.168.0.114 icmp_seq=3026 Destination Host Unreachable

As such, decided to set of on a mission to improve the performance of all my network routers.  Here are the settings used:

Shortcut Forwarding Engine: CTF ( If this results in OOM, use SFE )
STP: Disabled

MTU:    Manual    1452

# nvram show|grep -Ei ctf_fa_cap=1
size: 55223 bytes (75849 left)
ctf_fa_cap=1

NOTE: The last option may or may not be ideal for your router, depending if Flow Acceleration (FA) module is included in your setup and your router supports it.

Additionally, also scan the DD-WRT remote logs (You did setup rsyslog to a remote server right?) which can tell you, amongst other things excessive requests or packet storms and how many DNS queries occurred in 5 minutes (this is alot):

# grep -Ei "DPT=53" dd-wrt-inet.mds.xyz.log|wc -l
4155
#

Or 30K in 3 hours.  That's alot:

# grep -Ei "DPT=53" dd-wrt-inet.mds.xyz.log | wc -l
30022
#

To solve the above DNS queries problem, you can either tune the DNS masquerade on DD-WRT, if you use it, or adjust the DNS caching on your internal DNS servers.  Here's a Windows Server example:

PS C:\Users\Administrator.WINAD01.000> Get-DnsServerCache
MaxTTL                           : 1.00:00:00
MaxNegativeTTL                   : 00:15:00
MaxKBSize                        : 0
EnablePollutionProtection        : True
LockingPercent                   : 100
StoreEmptyAuthenticationResponse : True

PS C:\Users\Administrator.WINAD01.000> Set-DnsServerCache -MaxKBSize 65536
PS C:\Users\Administrator.WINAD01.000> Get-DnsServerCache
MaxTTL                           : 1.00:00:00
MaxNegativeTTL                   : 00:15:00
MaxKBSize                        : 65536
EnablePollutionProtection        : True
LockingPercent                   : 100
StoreEmptyAuthenticationResponse : True

PS C:\Users\Administrator.WINAD01.000> Set-DnsServerCache -MaxKBSize 65536 -MaxTtl 0x15180
WARNING: The input value for the setting MaxTtl is lesser than a second and will be ignored.  The input value must be
in the format DD.HH:MM:SS where DD is days, HH is hours, MM is minutes and SS is seconds.

PS C:\Users\Administrator.WINAD01.000> Set-DnsServerCache -MaxKBSize 65536 -MaxTtl 2.00:00:00
PS C:\Users\Administrator.WINAD01.000> Get-DnsServerCache


MaxTTL                           : 2.00:00:00
MaxNegativeTTL                   : 00:15:00
MaxKBSize                        : 65536
EnablePollutionProtection        : True
LockingPercent                   : 100
StoreEmptyAuthenticationResponse : True

PS C:\Users\Administrator.WINAD01.000>

To set it to 2 days and something other then 0, which effectively, it seems, would turn this off.  Likewise for FreeIPA / IDM, use the following to adjust the DNS cache: 

NOTE: A word about OOM when using CTF.  Appears these OOM messages followed by reboots on one of the routers prompted me to change back to SFE:

# cat dd-wrt-roma.mds.xyz.log|grep -Ei oom_kill
Nov 21 02:54:27 dd-wrt-roma.mds.xyz kernel: [20094.748505] [<80014094>] (dump_header) from [<800b7344>] (oom_kill_process+0xec/0x3cc)
Nov 21 02:54:27 dd-wrt-roma.mds.xyz kernel: [20094.766870] [<800b7258>] (oom_kill_process) from [<800b78f0>] (out_of_memory+0x260/0x344)
Nov 21 04:06:13 dd-wrt-roma.mds.xyz kernel: [ 4191.173207] [<80014094>] (dump_header) from [<800b7344>] (oom_kill_process+0xec/0x3cc)
.
.
.
Nov 23 22:15:56 dd-wrt-roma.mds.xyz kernel: [47881.130510] [<80014094>] (dump_header) from [<800b7344>] (oom_kill_process+0xec/0x3cc)
Nov 23 22:15:56 dd-wrt-roma.mds.xyz kernel: [47881.148868] [<800b7258>] (oom_kill_process) from [<800b78f0>] (out_of_memory+0x260/0x344)
#
# cat dd-wrt-inet.mds.xyz.log|grep -Ei oom_kill
Nov 25 03:21:21 dd-wrt-inet.mds.xyz kernel: [172011.430393] [<80014094>] (dump_header) from [<800b7344>] (oom_kill_process+0xec/0x3cc)
Nov 26 04:07:38 dd-wrt-inet.mds.xyz kernel: [89063.941579] [<80014094>] (dump_header) from [<800b7344>] (oom_kill_process+0xec/0x3cc)
Nov 26 04:07:38 dd-wrt-inet.mds.xyz kernel: [89063.941594] [<800b7258>] (oom_kill_process) from [<800b78f0>] (out_of_memory+0x260/0x344)
#

EDIT: Nov 27th 2022

Appears that networking topology has alot to do with the performance as well as DNS caching above.  See posts below:

https://i2.wp.com/www.microdevsys.com/WordPressImages/NetworkTopologyNov27-2022.PNG?ssl=1

Cheers,
TK

REF: 
https://forum.dd-wrt.com/phpBB2/viewtopic.php?p=1275963#1275963
https://wiki.dd-wrt.com/wiki/index.php/Hardware#Flow_Acceleration.2C_SFE_and_Cut-Through_Forwarding
https://forum.dd-wrt.com/phpBB2/viewtopic.php?t=329793&sid=86de7e78395bca1b71a162cf5b82c1ef
https://forum.dd-wrt.com/phpBB2/viewtopic.php?p=85265
https://learn.microsoft.com/lb-lu/powershell/module/dnsserver/set-dnsservercache?view=winserver2012-ps
https://learn.microsoft.com/lb-lu/powershell/module/dnsserver/get-dnsservercache?view=winserver2012-ps

https://forum.dd-wrt.com/phpBB2/viewtopic.php?p=1276305#1276305

https://forum.dd-wrt.com/phpBB2/viewtopic.php?p=1276303#1276303 

Patroni and ETCD: Upgrade from PostgreSQL 10 to PostgreSQL 14

Patroni and ETCD: Upgrade from PostgreSQL 10 to PostgreSQL 14

Begin by installing PostgreSQL 14.  In this case we're installing all PostgreSQL 14 packages.

yum install postgresql14*

Check version after installation:

[root@psql04 ~]# /usr/pgsql-14/bin/psql –version
psql (PostgreSQL) 14.5
[root@psql04 ~]#

[root@psql04 ~]# grep -Ei bin_dir /etc/patroni.yml
    bin_dir: /usr/pgsql-14/bin
[root@psql04 ~]#

Read the rest of this entry »

Internal Database Error encountered: Could not connect to LDAP server host idmipa01.nix.mds.xyz port 636 Error netscape.ldap.LDAPException: Authentication failed (48)

Restore VM's from snapshot.  Yes, this is a new attempt at restoring some FreeIPA hosts that have been, ahem, neglected slightly to the point where things expired and don't work.  A few unexpected reboots and FS corruption didn't help the matter either.  Regardless, the recovery will in many ways show off the restoration capabilities of.FreeIPA which have certinly grew with the product.  Once again we see the following in the debug logs:

# tail -f /var/log/pki/pki-tomcat/ca/debug -n 200
Could not connect to LDAP server host idmipa01.nix.mds.xyz port 636 Error netscape.ldap.LDAPException: Authentication failed (48)
Internal Database Error encountered: Could not connect to LDAP server host idmipa01.nix.mds.xyz port 636 Error netscape.ldap.LDAPException: Authentication failed (48)
        at com.netscape.cmscore.dbs.DBSubsystem.init(DBSubsystem.java:689)

Use idmipa01 to fix certificates.  Set idmipa01 as renewal master, if not already:

# ipa config-mod \
      –ca-renewal-master-server idmipa01.nix.mds.xyz \
      | grep 'CA renewal master'

Set idmipa02 as following the renewal master (idmipa01 is designated / defacto master in the cluster)

[ idmipa01 ]
# ipa config-show | grep 'IPA CA renewal master'
  IPA CA renewal master: idmipa02.nix.mds.xyz

[ idmipa02 ]
# ipa config-show | grep 'IPA CA renewal master'
  IPA CA renewal master: idmipa02.nix.mds.xyz

[ idmipa02 ]
# ipa config-mod \
      –ca-renewal-master-server idmipa01.nix.mds.xyz \
      | grep 'CA renewal master'

Once this is done, certs appear with expiration dates as below:

# getcert list|grep -Ei expire
        expires: 2022-09-12 03:14:57 UTC
        expires: 2020-10-03 20:04:58 UTC
        expires: 2022-09-12 03:13:58 UTC
        expires: 2036-11-21 07:32:02 UTC
        expires: 2022-09-12 03:13:48 UTC
        expires: 2022-09-12 03:13:47 UTC
        expires: 2022-10-05 23:00:29 UTC
        expires: 2022-10-05 23:00:59 UTC
        expires: 2023-09-26 00:54:45 UTC

Start the IPA service ignoring failures:

# ipactl restart –ignore-service-failure

Follow steps on this RH blog:

https://access.redhat.com/solutions/3357261

# systemctl stop ntpd

# for nickname in "auditSigningCert cert-pki-ca" "ocspSigningCert cert-pki-ca" "subsystemCert cert-pki-ca" "Server-Cert cert-pki-ca"
do
  certdate=$(date -d "`certutil -L -d /etc/pki/pki-tomcat/alias -n "${nickname}" | grep -i after | cut -d: -f2-`" +%s )
  echo "$nickname – $certdate"
  [[ ${newdate:-99999999999} -gt “${certdate}” ]] && newdate=$certdate
done
date –set="`date –date=@$[newdate – 86400]`"

# systemctl restart certmonger

We are greeted with the following since the site certificate is valid only in the future:

# getcert list|grep -Ei expire
        expires: 2020-10-03 20:05:47 UTC
        expires: 2020-10-03 20:04:58 UTC
        expires: 2022-09-12 03:13:58 UTC
        expires: 2036-11-21 07:32:02 UTC
        expires: 2022-09-12 03:13:48 UTC
        expires: 2022-09-12 03:13:47 UTC
        ca-error: Server at https://idmipa01.nix.mds.xyz/ipa/xml failed request, will retry: -504 (libcurl failed to execute the HTTP POST transaction, explaining:  Peer's Certificate has expired.).
        expires: 2022-10-05 23:00:29 UTC
        ca-error: Server at https://idmipa01.nix.mds.xyz/ipa/xml failed request, will retry: -504 (libcurl failed to execute the HTTP POST transaction, explaining:  Peer's Certificate has expired.).
        expires: 2022-10-05 23:00:59 UTC
        expires: 2023-09-26 00:54:45 UTC

# openssl s_client -showcerts -connect idmipa01.nix.mds.xyz:443
CONNECTED(00000003)
depth=1 O = NIX.MDS.XYZ, CN = Certificate Authority
verify return:1
depth=0 O = NIX.MDS.XYZ, CN = idmipa01.nix.mds.xyz
verify error:num=9:certificate is not yet valid
notBefore=Oct  4 23:00:59 2020 GMT
verify return:1
depth=0 O = NIX.MDS.XYZ, CN = idmipa01.nix.mds.xyz
notBefore=Oct  4 23:00:59 2020 GMT
verify return:1

Certificate chain
 0 s:/O=NIX.MDS.XYZ/CN=idmipa01.nix.mds.xyz
   i:/O=NIX.MDS.XYZ/CN=Certificate Authority

[ …. ]

We notice that the date on the host was set to:

Fri Oct  2 20:12:43 EDT 2020

which is pior to the earliest date in the certificates:

# getcert list|grep -Ei expire
        expires: 2020-10-03 20:05:47 UTC
        expires: 2020-10-03 20:04:58 UTC

However, the Apache / HTTPD SSL Certificate is only valid after:

notBefore=Oct  4 23:00:59 2020 GMT

So we either need to update the HTTPD certificate or move the date past Oct 4th 2020.  Let's set the date to Oct 4th: 

# for nickname in "auditSigningCert cert-pki-ca" "ocspSigningCert cert-pki-ca" "subsystemCert cert-pki-ca" "Server-Cert cert-pki-ca"
do
  certdate=$(date -d "`certutil -L -d /etc/pki/pki-tomcat/alias -n "${nickname}" | grep -i after | cut -d: -f2-`" +%s )
  echo "$nickname – $certdate"
  [[ ${newdate:-99999999999} -gt “${certdate}” ]] && newdate=$certdate
done
date –set="`date –date=@$[newdate + 172800]`"

Restart certmonger and check status:

# systemctl restart certmonger

check status:

# getcert list

Still expired.  Did some reading:

https://frasertweedale.github.io/blog-redhat/posts/2019-05-24-ipa-cert-fix.html

and decided to try:

# ipa-cert-fix

                          WARNING

ipa-cert-fix is intended for recovery when expired certificates
prevent the normal operation of IPA.  It should ONLY be used
in such scenarios, and backup of the system, especially certificates
and keys, is STRONGLY RECOMMENDED.


The following certificates will be renewed:

Dogtag ca_ocsp_signing certificate:
  Subject: CN=OCSP Subsystem,O=NIX.MDS.XYZ
  Serial:  17
  Expires: 2020-10-03 20:04:58

Dogtag ca_audit_signing certificate:
  Subject: CN=CA Audit,O=NIX.MDS.XYZ
  Serial:  15
  Expires: 2020-10-03 20:05:47

Enter "yes" to proceed: yes
Proceeding.
Renewed Dogtag ca_ocsp_signing certificate:
  Subject: CN=OCSP Subsystem,O=NIX.MDS.XYZ
  Serial:  31
  Expires: 2022-09-26 00:11:14

Renewed Dogtag ca_audit_signing certificate:
  Subject: CN=CA Audit,O=NIX.MDS.XYZ
  Serial:  32
  Expires: 2022-09-26 00:12:16

Becoming renewal master.
The ipa-cert-fix command was successful


Which was apparently successful though failed to renew things:

# getcert list|grep -Ei expire
        expires: 2020-10-03 20:05:47 UTC *
        expires: 2020-10-03 20:04:58 UTC *
        expires: 2022-09-12 03:13:58 UTC
        expires: 2036-11-21 07:32:02 UTC
        expires: 2022-09-12 03:13:48 UTC
        expires: 2022-09-12 03:13:47 UTC
        expires: 2022-10-05 23:00:29 UTC
        expires: 2022-10-05 23:00:59 UTC
        expires: 2023-09-26 00:54:45 UTC

Restart certmonger which now captures correct dates:

# systemctl restart certmonger
# getcert list|grep -Ei expire

        expires: 2022-09-26 00:12:16 UTC *
        expires: 2022-09-26 00:11:14 UTC *
        expires: 2022-09-12 03:13:58 UTC
        expires: 2036-11-21 07:32:02 UTC
        expires: 2022-09-12 03:13:48 UTC
        expires: 2022-09-12 03:13:47 UTC
        expires: 2022-10-05 23:00:29 UTC
        expires: 2022-10-05 23:00:59 UTC
        expires: 2023-09-26 00:54:45 UTC

Restart IPA services ignoring failures in the process, while still maintaining the reset date of Oct 5th 2020 (Today is Sep 25 2022)

# ipactl restart –ignore-service-failure
Restarting Directory Service
Restarting krb5kdc Service
Restarting kadmin Service
Restarting named Service
Restarting httpd Service
Restarting ipa-custodia Service
Restarting ntpd Service
Restarting pki-tomcatd Service
Restarting smb Service
Restarting winbind Service
Restarting ipa-otpd Service
Restarting ipa-dnskeysyncd Service
ipa: INFO: The ipactl command was successful

This is the part where I realize the system was using the hwclcok date not the 'date' date (facepalm):

# date
Mon Oct  5 20:18:46 EDT 2020
# hwclock
Sun 25 Sep 2022 09:35:45 PM EDT  -0.321095 seconds

Hence why cert dates came back with 2022.  Whatever, let's set the date back.  It may work but let's check the UI certs.  Copy the openssl output certificate portions from above into a file and run the following:

# cat site-cert.pem
—–BEGIN CERTIFICATE—–
MIIEmzCCA4OgAwI…………………………………………………HIFvjW5pjp58mflhQ==
—–END CERTIFICATE—–
 1 s:/O=NIX.MDS.XYZ/CN=Certificate Authority
   i:/O=NIX.MDS.XYZ/CN=Certificate Authority
—–BEGIN CERTIFICATE—–
MIIDkDCC……………………………………………………………..w0T37yu7pbxM
LGclqw==
—–END CERTIFICATE—–

Check the site cert extracted from the above command:

# openssl x509 -enddate -startdate -noout -in site-cert.pem
notAfter=Oct  5 23:00:59 2022 GMT
notBefore=Oct  4 23:00:59 2020 GMT

Cert appears good until 2022 Oct 5th which we are not yet in.  Let's set the date forwards a tad:

# hwclock –set –date "Fri Sep 25 21:49:00 EDT 2022"; date -s "Fri Sep 25 21:49:00 EDT 2022"
# systemctl restart ntpd
# ntpdate -s 192.168.0.12                                                                            # My NTP host.

Now try a status and a restart as well:

# ipactl status
Directory Service: RUNNING
krb5kdc Service: RUNNING
kadmin Service: RUNNING
named Service: RUNNING
httpd Service: RUNNING
ipa-custodia Service: RUNNING
ntpd Service: RUNNING
pki-tomcatd Service: RUNNING
smb Service: RUNNING
winbind Service: RUNNING
ipa-otpd Service: RUNNING
ipa-dnskeysyncd Service: RUNNING
ipa: INFO: The ipactl command was successful

But checking the certs again, seeing two more that are older then Sep 25 2022:

# getcert list|grep -Ei expire
        expires: 2022-09-26 00:12:16 UTC
        expires: 2022-09-26 00:11:14 UTC 
        expires: 2022-09-12 03:13:58 UTC *
        expires: 2036-11-21 07:32:02 UTC
        expires: 2022-09-12 03:13:48 UTC *
        expires: 2022-09-12 03:13:47 UTC *
        expires: 2022-10-05 23:00:29 UTC
        expires: 2022-10-05 23:00:59 UTC
        expires: 2023-09-26 00:54:45 UTC

Need to move the dates back again to a day prior and renew again:

# for nickname in "auditSigningCert cert-pki-ca" "ocspSigningCert cert-pki-ca" "subsystemCert cert-pki-ca" "Server-Cert cert-pki-ca"
do
  certdate=$(date -d "`certutil -L -d /etc/pki/pki-tomcat/alias -n "${nickname}" | grep -i after | cut -d: -f2-`" +%s )
  echo "$nickname – $certdate"
  [[ ${newdate:-99999999999} -gt “${certdate}” ]] && newdate=$certdate
done
date –set="`date –date=@$[newdate + 86400]`"

Well that above command failed:

Sun Oct  4 20:04:58 EDT 2020

There is no certs with that date:

# getcert list|grep -Ei expire
        expires: 2022-09-26 00:12:16 UTC
        expires: 2022-09-26 00:11:14 UTC
        expires: 2022-09-12 03:13:58 UTC
        expires: 2036-11-21 07:32:02 UTC
        expires: 2022-09-12 03:13:48 UTC
        expires: 2022-09-12 03:13:47 UTC
        expires: 2022-10-05 23:00:29 UTC
        expires: 2022-10-05 23:00:59 UTC
        expires: 2023-09-26 00:54:45 UTC

So let's try a modified copy:

# for nickname in "auditSigningCert cert-pki-ca" "ocspSigningCert cert-pki-ca" "subsystemCert cert-pki-ca" "Server-Cert cert-pki-ca"
do
  certdate=$(certutil -L -d /etc/pki/pki-tomcat/alias -n "${nickname}" | grep -i after)
  echo $certdate;
done

Somehow this script is basing this off of the current date?  I won't reverse engineer it and set the date manually instead:

# hwclock –set –date "Fri Sep 12 01:00:00 EDT 2022"; date -s "Fri Sep 12 01:00:00 EDT 2022"

You should see certificates in submitting status now:

# getcert list
Number of certificates and requests being tracked: 9.
Request ID '20180122053031':
        status: SUBMITTING
        stuck: no
        key pair storage: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='auditSigningCert cert-pki-ca',token='NSS Certificate DB',pin set
        certificate: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='auditSigningCert cert-pki-ca',token='NSS Certificate DB'
        CA: dogtag-ipa-ca-renew-agent
        issuer: CN=Certificate Authority,O=NIX.MDS.XYZ
        subject: CN=CA Audit,O=NIX.MDS.XYZ
        expires: 2022-09-26 00:12:16 UTC
        key usage: digitalSignature,nonRepudiation
        pre-save command: /usr/libexec/ipa/certmonger/stop_pkicad
        post-save command: /usr/libexec/ipa/certmonger/renew_ca_cert "auditSigningCert cert-pki-ca"
        track: yes
        auto-renew: yes
Request ID '20180122053032':
        status: MONITORING
        stuck: no
        key pair storage: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='ocspSigningCert cert-pki-ca',token='NSS Certificate DB',pin set
        certificate: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='ocspSigningCert cert-pki-ca',token='NSS Certificate DB'
        CA: dogtag-ipa-ca-renew-agent
        issuer: CN=Certificate Authority,O=NIX.MDS.XYZ
        subject: CN=OCSP Subsystem,O=NIX.MDS.XYZ
        expires: 2022-09-26 00:11:14 UTC
        eku: id-kp-OCSPSigning
        pre-save command: /usr/libexec/ipa/certmonger/stop_pkicad
        post-save command: /usr/libexec/ipa/certmonger/renew_ca_cert "ocspSigningCert cert-pki-ca"
        track: yes
        auto-renew: yes
Request ID '20180122053033':
        status: MONITORING
        stuck: no
        key pair storage: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='subsystemCert cert-pki-ca',token='NSS Certificate DB',pin set
        certificate: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='subsystemCert cert-pki-ca',token='NSS Certificate DB'
        CA: dogtag-ipa-ca-renew-agent
        issuer: CN=Certificate Authority,O=NIX.MDS.XYZ
        subject: CN=CA Subsystem,O=NIX.MDS.XYZ
        expires: 2022-09-12 03:13:58 UTC
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command: /usr/libexec/ipa/certmonger/stop_pkicad
        post-save command: /usr/libexec/ipa/certmonger/renew_ca_cert "subsystemCert cert-pki-ca"
        track: yes
        auto-renew: yes
Request ID '20180122053034':
        status: MONITORING
        stuck: no
        key pair storage: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='caSigningCert cert-pki-ca',token='NSS Certificate DB',pin set
        certificate: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='caSigningCert cert-pki-ca',token='NSS Certificate DB'
        CA: dogtag-ipa-ca-renew-agent
        issuer: CN=Certificate Authority,O=NIX.MDS.XYZ
        subject: CN=Certificate Authority,O=NIX.MDS.XYZ
        expires: 2036-11-21 07:32:02 UTC
        key usage: digitalSignature,nonRepudiation,keyCertSign,cRLSign
        pre-save command: /usr/libexec/ipa/certmonger/stop_pkicad
        post-save command: /usr/libexec/ipa/certmonger/renew_ca_cert "caSigningCert cert-pki-ca"
        track: yes
        auto-renew: yes
Request ID '20180122053035':
        status: SUBMITTING
        stuck: no
        key pair storage: type=FILE,location='/var/lib/ipa/ra-agent.key'
        certificate: type=FILE,location='/var/lib/ipa/ra-agent.pem'
        CA: dogtag-ipa-ca-renew-agent
        issuer: CN=Certificate Authority,O=NIX.MDS.XYZ
        subject: CN=IPA RA,O=NIX.MDS.XYZ
        expires: 2022-09-12 03:13:48 UTC
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command: /usr/libexec/ipa/certmonger/renew_ra_cert_pre
        post-save command: /usr/libexec/ipa/certmonger/renew_ra_cert
        track: yes
        auto-renew: yes
Request ID '20180122053036':
        status: MONITORING
        stuck: no
        key pair storage: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='Server-Cert cert-pki-ca',token='NSS Certificate DB',pin set
        certificate: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='Server-Cert cert-pki-ca',token='NSS Certificate DB'
        CA: dogtag-ipa-ca-renew-agent
        issuer: CN=Certificate Authority,O=NIX.MDS.XYZ
        subject: CN=idmipa01.nix.mds.xyz,O=NIX.MDS.XYZ
        expires: 2022-09-12 03:13:47 UTC
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command: /usr/libexec/ipa/certmonger/stop_pkicad
        post-save command: /usr/libexec/ipa/certmonger/renew_ca_cert "Server-Cert cert-pki-ca"
        track: yes
        auto-renew: yes
Request ID '20180122053037':
        status: SUBMITTING
        stuck: no
        key pair storage: type=NSSDB,location='/etc/dirsrv/slapd-NIX-MDS-XYZ',nickname='Server-Cert',token='NSS Certificate DB',pinfile='/etc/dirsrv/slapd-NIX-MDS-XYZ/pwdfile.txt'
        certificate: type=NSSDB,location='/etc/dirsrv/slapd-NIX-MDS-XYZ',nickname='Server-Cert',token='NSS Certificate DB'
        CA: IPA
        issuer: CN=Certificate Authority,O=NIX.MDS.XYZ
        subject: CN=idmipa01.nix.mds.xyz,O=NIX.MDS.XYZ
        expires: 2022-10-05 23:00:29 UTC
        principal name: ldap/idmipa01.nix.mds.xyz@NIX.MDS.XYZ
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command:
        post-save command: /usr/libexec/ipa/certmonger/restart_dirsrv NIX-MDS-XYZ
        track: yes
        auto-renew: yes
Request ID '20180122053042':
        status: SUBMITTING
        stuck: no
        key pair storage: type=NSSDB,location='/etc/httpd/alias',nickname='Server-Cert',token='NSS Certificate DB',pinfile='/etc/httpd/alias/pwdfile.txt'
        certificate: type=NSSDB,location='/etc/httpd/alias',nickname='Server-Cert',token='NSS Certificate DB'
        CA: IPA
        issuer: CN=Certificate Authority,O=NIX.MDS.XYZ
        subject: CN=idmipa01.nix.mds.xyz,O=NIX.MDS.XYZ
        expires: 2022-10-05 23:00:59 UTC
        principal name: HTTP/idmipa01.nix.mds.xyz@NIX.MDS.XYZ
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command:
        post-save command: /usr/libexec/ipa/certmonger/restart_httpd
        track: yes
        auto-renew: yes
Request ID '20180122053135':
        status: MONITORING
        stuck: no
        key pair storage: type=FILE,location='/var/kerberos/krb5kdc/kdc.key'
        certificate: type=FILE,location='/var/kerberos/krb5kdc/kdc.crt'
        CA: SelfSign
        issuer: CN=idmipa01.nix.mds.xyz,O=NIX.MDS.XYZ
        subject: CN=idmipa01.nix.mds.xyz,O=NIX.MDS.XYZ
        expires: 2023-09-26 00:54:45 UTC
        principal name: krbtgt/NIX.MDS.XYZ@NIX.MDS.XYZ
        certificate template/profile: KDCs_PKINIT_Certs
        pre-save command:
        post-save command: /usr/libexec/ipa/certmonger/renew_kdc_cert
        track: yes
        auto-renew: yes

Wait a bit and check again if they were successfully processed.  Or not:

ca-error: Server at https://idmipa01.nix.mds.xyz/ipa/xml failed request, will retry: 907 (RPC failed at server.  cannot connect to 'https://idmipa01.nix.mds.xyz:443/ca/rest/account/login': [SSL: SSL_HANDSHAKE_FAILURE] ssl handshake failure (_ssl.c:1826)).

Let's try the following command again:

# ipa-cert-fix -v

But we get this instead:

INFO: Restoring previous LDAP configuration
ERROR: Unable to find CSR for sslserver cert

ipapython.admintool: DEBUG:   File "/usr/lib/python2.7/site-packages/ipapython/admintool.py", line 178, in execute
    return_value = self.run()
  File "/usr/lib/python2.7/site-packages/ipaserver/install/ipa_cert_fix.py", line 128, in run
    replicate_dogtag_certs(subject_base, ca_subject_dn, certs)
  File "/usr/lib/python2.7/site-packages/ipaserver/install/ipa_cert_fix.py", line 251, in replicate_dogtag_certs
    cert = x509.load_certificate_from_file(cert_path)
  File "/usr/lib/python2.7/site-packages/ipalib/x509.py", line 425, in load_certificate_from_file
    with open(filename, mode='rb') as f:

ipapython.admintool: DEBUG: The ipa-cert-fix command failed, exception: IOError: [Errno 2] No such file or directory: '/etc/pki/pki-tomcat/certs/sslserver.crt'
ipapython.admintool: ERROR: [Errno 2] No such file or directory: '/etc/pki/pki-tomcat/certs/sslserver.crt'
ipapython.admintool: ERROR: The ipa-cert-fix command failed.


And we fix with this article:

https://access.redhat.com/solutions/4852721

Following the document steps, convert the cert accordingly:

# grep -A 19 csr /var/lib/certmonger/requests/20180122053033
csr=—–BEGIN NEW CERTIFICATE REQUEST—–
 MIIDJTCCAg………………………………………………X1cWBn+CU=
 —–END NEW CERTIFICATE REQUEST—–
spkac=MIICQDCCASgwgg…………….p78JfKV2/VHxXJTULg==
# vi 1.txt

# cat 1.txt
ca.subsystem.certreq=MIIDJTCCAg0CAQA…………….45oAX1cWBn+CU=

Make backups of anything you modify, whether or not you have snapshots. (Snapshots will cause you to restart from the beginning of this article 😛 )

# cp -ip /etc/pki/pki-tomcat/ca/CS.cfg /etc/pki/pki-tomcat/ca/CS.cfg-backup01

Confirm if the line exists:

# grep -Ei ca.subsystem.certreq /etc/pki/pki-tomcat/ca/CS.cfg

It should not otherwise you wouldn't get the above error:

Add the ca.subsystem.certreq= below the ca.subsystem.cert= line in /etc/pki/pki-tomcat/ca/CS.cfg:

# grep -Ei ca.subsystem.certreq /etc/pki/pki-tomcat/ca/CS.cfg
ca.subsystem.certreq=MIIDJTCCAg0CAQ………………………..X1cWBn+CU=

Let's try the command again:

# ipa-cert-fix

But no luck:

# ipa-cert-fix
[ ….. ]
Enter "yes" to proceed: yes
Proceeding.
[Errno 2] No such file or directory: '/etc/pki/pki-tomcat/certs/sslserver.crt'
The ipa-cert-fix command failed.

Let's move the dates back again, manually:

# hwclock –set –date "Fri Sep 12 01:00:00 EDT 2022"; date -s "Fri Sep 12 01:00:00 EDT 2022"

# ipa-cert-fix
[ ….. ]
Enter "yes" to proceed: yes
Proceeding.
Command 'pki-server cert-fix –ldapi-socket /var/run/slapd-NIX-MDS-XYZ.socket –agent-uid ipara –cert sslserver –cert subsystem –cert ca_ocsp_signing –cert ca_audit_signing –extra-cert 25' returned non-zero exit status 1
The ipa-cert-fix command failed.

did not work.  So moving slightly ahead:

# hwclock –set –date "Fri Sep 13 01:00:00 EDT 2022"; date -s "Fri Sep 13 01:00:00 EDT 2022"

results in absolutely nothing.  So trying with a different date:

# hwclock –set –date "Fri Sep 11 04:00:00 EDT 2022"; date -s "Fri Sep 11 04:00:00 EDT 2022"
Sun Sep 11 04:00:00 EDT 2022

Resulted in a successfully started host:

# ipactl restart –ignore-service-failure
Restarting Directory Service
Restarting krb5kdc Service
Restarting kadmin Service
Restarting named Service
Restarting httpd Service
Restarting ipa-custodia Service
Restarting ntpd Service
Restarting pki-tomcatd Service
Restarting smb Service
Restarting winbind Service
Restarting ipa-otpd Service
Restarting ipa-dnskeysyncd Service
ipa: INFO: The ipactl command was successful

Hmm, ok we're on to something here.  Now that the services started fully, let's use the following:

# ipa-cert-fix

                          WARNING

ipa-cert-fix is intended for recovery when expired certificates
prevent the normal operation of IPA.  It should ONLY be used
in such scenarios, and backup of the system, especially certificates
and keys, is STRONGLY RECOMMENDED.


The following certificates will be renewed:

Dogtag sslserver certificate:
  Subject: CN=idmipa01.nix.mds.xyz,O=NIX.MDS.XYZ
  Serial:  22
  Expires: 2022-09-12 03:13:47

Dogtag subsystem certificate:
  Subject: CN=CA Subsystem,O=NIX.MDS.XYZ
  Serial:  26
  Expires: 2022-09-12 03:13:58

IPA IPA RA certificate:
  Subject: CN=IPA RA,O=NIX.MDS.XYZ
  Serial:  25
  Expires: 2022-09-12 03:13:48

Enter "yes" to proceed: yes
Proceeding.
Command 'pki-server cert-fix –ldapi-socket /var/run/slapd-NIX-MDS-XYZ.socket –agent-uid ipara –cert sslserver –cert subsystem –extra-cert 25' returned non-zero exit status 1
The ipa-cert-fix command failed.

# pki-server cert-fix –ldapi-socket /var/run/slapd-NIX-MDS-XYZ.socket –agent-uid ipara –cert sslserver –cert subsystem –extra-cert 25
INFO: Loading password config: /etc/pki/pki-tomcat/password.conf
INFO: Fixing the following system certs: [‘sslserver’, ‘subsystem’]
INFO: Renewing the following additional certs: [’25’]
SASL/EXTERNAL authentication started
SASL username: gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth
SASL SSF: 0
INFO: Stopping the instance to proceed with system cert renewal
INFO: Configuring LDAP password authentication
INFO: Setting pkidbuser password via ldappasswd
SASL/EXTERNAL authentication started
SASL username: gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth
SASL SSF: 0
INFO: Selftests disabled for subsystems: ca
INFO: Resetting password for uid=ipara,ou=people,o=ipaca
SASL/EXTERNAL authentication started
SASL username: gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth
SASL SSF: 0
INFO: Creating a temporary sslserver cert
INFO: Getting sslserver cert info for ca
INFO: Trying to create a new temp cert for sslserver.
INFO: Generate temp SSL certificate
INFO: Getting sslserver cert info for ca
INFO: Selftests enabled for subsystems: ca
INFO: Restoring previous LDAP configuration
ERROR: Unable to find CSR for sslserver cert

Looks like it expects more CSR's.  In this case:

Request ID '20180122053036':
        certificate: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='Server-Cert cert-pki-ca',token='NSS Certificate DB'
        expires: 2022-09-12 03:13:47 UTC

Doesn't have a CSR, so we add one`:

# cat /var/lib/certmonger/requests/20180122053036
csr=—–BEGIN NEW CERTIFICATE REQUEST—–
 MIIDIzCCAgsCA……………………………………xQ/FFfh

Then convert it to the following one liner in the scrxipt with nickname ca.sslserver.certreq in this case:

ca.sslserver.certreq=MIIDIzCCAgsCA…………………………………………………………5XsHg07A8

But, alas, I had a copy in another cert:

csr=—–BEGIN NEW CERTIFICATE REQUEST—–
 MIIDIzCC……………………………………………..kxQ/FFfh
 —–END NEW CERTIFICATE REQUEST—–

# grep certreq /etc/pki/pki-tomcat/ca/CS.cfg
ca.sslserver.certreq=MIIDIzCCAgsCA……………………………………………………………………..V5XsHg07A8

NOTE the missing FFfh characters in the CSR vs what I typed in the CS.cfg.  Hence got this:

# ipa-cert-fix -v
[ …………… ]
SASL SSF: 0
INFO: Creating a temporary sslserver cert
INFO: Getting sslserver cert info for ca
INFO: Trying to create a new temp cert for sslserver.
INFO: Generate temp SSL certificate
INFO: Getting sslserver cert info for ca
INFO: CSR for sslserver has been written to /tmp/tmpYQSMJk/sslserver.csr
INFO: Getting signing cert info for ca
INFO: CA cert written to /tmp/tmpYQSMJk/ca_certificate.crt
INFO: AKI: 0x1F737CF691BC6D8F93ACA3599FB6DBAB35AED71D
INFO: Selftests enabled for subsystems: ca
INFO: Restoring previous LDAP configuration
ERROR: Failed to generate CA-signed temp SSL certificate. RC: 255

ipapython.admintool: DEBUG:   File "/usr/lib/python2.7/site-packages/ipapython/admintool.py", line 178, in execute
    return_value = self.run()
  File "/usr/lib/python2.7/site-packages/ipaserver/install/ipa_cert_fix.py", line 117, in run
    run_cert_fix(certs, extra_certs)
  File "/usr/lib/python2.7/site-packages/ipaserver/install/ipa_cert_fix.py", line 245, in run_cert_fix
    ipautil.run(cmd, raiseonerr=True)
  File "/usr/lib/python2.7/site-packages/ipapython/ipautil.py", line 563, in run
    raise CalledProcessError(p.returncode, arg_string, str(output))

ipapython.admintool: DEBUG: The ipa-cert-fix command failed, exception: CalledProcessError: Command 'pki-server cert-fix –ldapi-socket /var/run/slapd-NIX-MDS-XYZ.socket –agent-uid ipara –cert sslserver –cert subsystem –extra-cert 25' returned non-zero exit status 1
ipapython.admintool: ERROR: Command 'pki-server cert-fix –ldapi-socket /var/run/slapd-NIX-MDS-XYZ.socket –agent-uid ipara –cert sslserver –cert subsystem –extra-cert 25' returned non-zero exit status 1
ipapython.admintool: ERROR: The ipa-cert-fix command failed.

Editing and ensuring it's correct this time, using a one liner to properly set it up:

# grep -A 19 csr /var/lib/certmonger/requests/20180122053036|grep -v spkac|grep -v "-"|tr '\n' ' '|sed -e "s/ //g"
MIIDIzCCAgs…………………………………………………………….zHkxQ/FFfh

seams to have allowed IPA to restart properly:

# ipa-cert-fix -v
[ ………………….. ]
INFO: Starting the instance with renewed certs

Renewed Dogtag sslserver certificate:
  Subject: CN=idmipa01.nix.mds.xyz,O=NIX.MDS.XYZ
  Serial:  34
  Expires: 2024-08-31 09:03:43

Renewed Dogtag subsystem certificate:
  Subject: CN=CA Subsystem,O=NIX.MDS.XYZ
  Serial:  35
  Expires: 2024-08-31 09:03:43

Renewed IPA IPA RA certificate:
  Subject: CN=IPA RA,O=NIX.MDS.XYZ
  Serial:  36
  Expires: 2024-08-31 09:03:44

ipalib.backend: DEBUG: Created connection context.ldap2_139668384537744
ipalib.backend: DEBUG: Destroyed connection context.ldap2_139668384537744
Becoming renewal master.
ipalib.install.sysrestore: DEBUG: Loading StateFile from '/var/lib/ipa/sysrestore/sysrestore.state'
ipalib.install.sysrestore: DEBUG: Loading Index file from '/var/lib/ipa/sysrestore/sysrestore.index'
ipapython.ipautil: DEBUG: Starting external process
ipapython.ipautil: DEBUG: args=ipactl restart
ipapython.ipautil: DEBUG: Process finished, return code=0
ipapython.ipautil: DEBUG: stdout=Restarting Directory Service
Restarting krb5kdc Service
Restarting kadmin Service
Restarting named Service
Restarting httpd Service
Restarting ipa-custodia Service
Restarting ntpd Service
Restarting pki-tomcatd Service
Restarting smb Service
Restarting winbind Service
Restarting ipa-otpd Service
Restarting ipa-dnskeysyncd Service

ipapython.ipautil: DEBUG: stderr=ipa: INFO: The ipactl command was successful

ipapython.admintool: INFO: The ipa-cert-fix command was successful

yet no change to the certs above.  Trying the renew option now:

# ipa-cacert-manage renew

Following this page:

https://floblanc.wordpress.com/2016/12/19/troubleshooting-certmonger-issues-with-freeipa/

ran the following:

# getcert modify-ca -c dogtag-ipa-ca-renew-agent -e '/usr/libexec/certmonger/dogtag-ipa-ca-renew-agent-submit -vv'

java.lang.Exception: Certutils.verifySystemCertValidityByNickname:  faliled: nickname: ocspSigningCert cert-pki-cacause: java.lang.Exception: Certutils.verifySystemCertValidityByNickname:  failed: nickname: ocspSigningCert cert-pki-ca

[11/Sep/2022:23:10:10][localhost-startStop-1]: SignedAuditLogger: event SELFTESTS_EXECUTION
[11/Sep/2022:23:10:10][localhost-startStop-1]: SelfTestSubsystem: Shutting down server due to selftest failure: Certutils.verifySystemCertValidityByNickname:  faliled: nickname: ocspSigningCert cert-pki-cacause: java.lang.Exception: Certutils.verifySystemCertValidityByNickname:  failed: nickname: ocspSigningCert cert-pki-ca
java.lang.Exception: Certutils.verifySystemCertValidityByNickname:  faliled: nickname: ocspSigningCert cert-pki-cacause: java.lang.Exception: Certutils.verifySystemCertValidityByNickname:  failed: nickname: ocspSigningCert cert-pki-ca

The below errors could have been when IPA services were stopped while the ipactl restart command was executed:

Request ID '20180122053035':
        status: CA_UNREACHABLE
        ca-error: Error 7 connecting to http://idmipa01.nix.mds.xyz:8080/ca/ee/ca/profileSubmit: Couldn't connect to server.
        stuck: no
        key pair storage: type=FILE,location='/var/lib/ipa/ra-agent.key'
        certificate: type=FILE,location='/var/lib/ipa/ra-agent.pem'
        CA: dogtag-ipa-ca-renew-agent
        issuer: CN=Certificate Authority,O=NIX.MDS.XYZ
        subject: CN=IPA RA,O=NIX.MDS.XYZ
        expires: 2022-09-12 03:13:48 UTC
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command: /usr/libexec/ipa/certmonger/renew_ra_cert_pre
        post-save command: /usr/libexec/ipa/certmonger/renew_ra_cert
        track: yes
        auto-renew: yes
Request ID '20180122053036':
        status: CA_UNREACHABLE
        ca-error: Error 7 connecting to http://idmipa01.nix.mds.xyz:8080/ca/ee/ca/profileSubmit: Couldn't connect to server.
        stuck: no
        key pair storage: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='Server-Cert cert-pki-ca',token='NSS Certificate DB',pin set
        certificate: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='Server-Cert cert-pki-ca',token='NSS Certificate DB'
        CA: dogtag-ipa-ca-renew-agent
        issuer: CN=Certificate Authority,O=NIX.MDS.XYZ
        subject: CN=idmipa01.nix.mds.xyz,O=NIX.MDS.XYZ
        expires: 2022-09-12 03:13:47 UTC
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command: /usr/libexec/ipa/certmonger/stop_pkicad
        post-save command: /usr/libexec/ipa/certmonger/renew_ca_cert "Server-Cert cert-pki-ca"
        track: yes
        auto-renew: yes

Tried resubmitting manually, perhaps the service was offline when it attempted upon ipactl restart execution:

# getcert resubmit -i 20180122053036
Resubmitting "20180122053036" to "dogtag-ipa-ca-renew-agent".

# getcert resubmit -i 20180122053035
Resubmitting "20180122053035" to "dogtag-ipa-ca-renew-agent".

And this time those two certs are ok:

Request ID '20180122053035':
        status: MONITORING
        stuck: no
        key pair storage: type=FILE,location='/var/lib/ipa/ra-agent.key'
        certificate: type=FILE,location='/var/lib/ipa/ra-agent.pem'
        CA: dogtag-ipa-ca-renew-agent
        issuer: CN=Certificate Authority,O=NIX.MDS.XYZ
        subject: CN=IPA RA,O=NIX.MDS.XYZ
        expires: 2024-08-31 09:03:44 UTC
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command: /usr/libexec/ipa/certmonger/renew_ra_cert_pre
        post-save command: /usr/libexec/ipa/certmonger/renew_ra_cert
        track: yes
        auto-renew: yes
Request ID '20180122053036':
        status: MONITORING
        stuck: no
        key pair storage: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='Server-Cert cert-pki-ca',token='NSS Certificate DB',pin set
        certificate: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='Server-Cert cert-pki-ca',token='NSS Certificate DB'
        CA: dogtag-ipa-ca-renew-agent
        issuer: CN=Certificate Authority,O=NIX.MDS.XYZ
        subject: CN=idmipa01.nix.mds.xyz,O=NIX.MDS.XYZ
        expires: 2024-08-31 09:03:43 UTC
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command: /usr/libexec/ipa/certmonger/stop_pkicad
        post-save command: /usr/libexec/ipa/certmonger/renew_ca_cert "Server-Cert cert-pki-ca"
        track: yes
        auto-renew: yes

This moves us forward to the last two:

Request ID '20180122053037':
        status: CA_UNREACHABLE
        ca-error: Server at https://idmipa01.nix.mds.xyz/ipa/xml failed request, will retry: 907 (RPC failed at server.  cannot connect to 'https://idmipa01.nix.mds.xyz:443/ca/rest/account/login': [SSL: SSL_HANDSHAKE_FAILURE] ssl handshake failure (_ssl.c:1826)).
        stuck: no
        key pair storage: type=NSSDB,location='/etc/dirsrv/slapd-NIX-MDS-XYZ',nickname='Server-Cert',token='NSS Certificate DB',pinfile='/etc/dirsrv/slapd-NIX-MDS-XYZ/pwdfile.txt'
        certificate: type=NSSDB,location='/etc/dirsrv/slapd-NIX-MDS-XYZ',nickname='Server-Cert',token='NSS Certificate DB'
        CA: IPA
        issuer: CN=Certificate Authority,O=NIX.MDS.XYZ
        subject: CN=idmipa01.nix.mds.xyz,O=NIX.MDS.XYZ
        expires: 2022-10-05 23:00:29 UTC
        principal name: ldap/idmipa01.nix.mds.xyz@NIX.MDS.XYZ
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command:
        post-save command: /usr/libexec/ipa/certmonger/restart_dirsrv NIX-MDS-XYZ
        track: yes
        auto-renew: yes
Request ID '20180122053042':
        status: CA_UNREACHABLE
        ca-error: Server at https://idmipa01.nix.mds.xyz/ipa/xml failed request, will retry: 907 (RPC failed at server.  cannot connect to 'https://idmipa01.nix.mds.xyz:443/ca/rest/account/login': [SSL: SSL_HANDSHAKE_FAILURE] ssl handshake failure (_ssl.c:1826)).
        stuck: no
        key pair storage: type=NSSDB,location='/etc/httpd/alias',nickname='Server-Cert',token='NSS Certificate DB',pinfile='/etc/httpd/alias/pwdfile.txt'
        certificate: type=NSSDB,location='/etc/httpd/alias',nickname='Server-Cert',token='NSS Certificate DB'
        CA: IPA
        issuer: CN=Certificate Authority,O=NIX.MDS.XYZ
        subject: CN=idmipa01.nix.mds.xyz,O=NIX.MDS.XYZ
        expires: 2022-10-05 23:00:59 UTC
        principal name: HTTP/idmipa01.nix.mds.xyz@NIX.MDS.XYZ
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command:
        post-save command: /usr/libexec/ipa/certmonger/restart_httpd
        track: yes
        auto-renew: yes
Request ID '20180122053135':
        status: MONITORING
        stuck: no
        key pair storage: type=FILE,location='/var/kerberos/krb5kdc/kdc.key'
        certificate: type=FILE,location='/var/kerberos/krb5kdc/kdc.crt'
        CA: SelfSign
        issuer: CN=idmipa01.nix.mds.xyz,O=NIX.MDS.XYZ
        subject: CN=idmipa01.nix.mds.xyz,O=NIX.MDS.XYZ
        expires: 2023-09-26 00:54:45 UTC
        principal name: krbtgt/NIX.MDS.XYZ@NIX.MDS.XYZ
        certificate template/profile: KDCs_PKINIT_Certs
        pre-save command:
        post-save command: /usr/libexec/ipa/certmonger/renew_kdc_cert
        track: yes
        auto-renew: yes

Let's repeat the resubmission for these 2 as well.  This time the error changed:

Request ID '20180122053037':
        status: CA_UNREACHABLE
        ca-error: Server at https://idmipa01.nix.mds.xyz/ipa/xml failed request, will retry: 4016 (RPC failed at server.  Failed to authenticate to CA REST API).

        stuck: no
        key pair storage: type=NSSDB,location='/etc/dirsrv/slapd-NIX-MDS-XYZ',nickname='Server-Cert',token='NSS Certificate DB',pinfile='/etc/dirsrv/slapd-NIX-MDS-XYZ/pwdfile.txt'
        certificate: type=NSSDB,location='/etc/dirsrv/slapd-NIX-MDS-XYZ',nickname='Server-Cert',token='NSS Certificate DB'
        CA: IPA
        issuer: CN=Certificate Authority,O=NIX.MDS.XYZ
        subject: CN=idmipa01.nix.mds.xyz,O=NIX.MDS.XYZ
        expires: 2022-10-05 23:00:29 UTC
        principal name: ldap/idmipa01.nix.mds.xyz@NIX.MDS.XYZ
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command:
        post-save command: /usr/libexec/ipa/certmonger/restart_dirsrv NIX-MDS-XYZ
        track: yes
        auto-renew: yes
Request ID '20180122053042':
        status: CA_UNREACHABLE
        ca-error: Server at https://idmipa01.nix.mds.xyz/ipa/xml failed request, will retry: 4016 (RPC failed at server.  Failed to authenticate to CA REST API).

        stuck: no
        key pair storage: type=NSSDB,location='/etc/httpd/alias',nickname='Server-Cert',token='NSS Certificate DB',pinfile='/etc/httpd/alias/pwdfile.txt'
        certificate: type=NSSDB,location='/etc/httpd/alias',nickname='Server-Cert',token='NSS Certificate DB'
        CA: IPA
        issuer: CN=Certificate Authority,O=NIX.MDS.XYZ
        subject: CN=idmipa01.nix.mds.xyz,O=NIX.MDS.XYZ
        expires: 2022-10-05 23:00:59 UTC
        principal name: HTTP/idmipa01.nix.mds.xyz@NIX.MDS.XYZ
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command:
        post-save command: /usr/libexec/ipa/certmonger/restart_httpd
        track: yes
        auto-renew: yes

Reading, this could help:

https://floblanc.wordpress.com/2016/12/19/troubleshooting-certmonger-issues-with-freeipa/

# ipa-certupdate
trying https://idmipa01.nix.mds.xyz/ipa/session/json
[try 1]: Forwarding 'ca_is_enabled/1' to json server 'https://idmipa01.nix.mds.xyz/ipa/session/json'
[try 1]: Forwarding 'ca_find/1' to json server 'https://idmipa01.nix.mds.xyz/ipa/session/json'
Systemwide CA database updated.
Systemwide CA database updated.
The ipa-certupdate command was successful

It appears to have done something.  Let's check what that is:

Request ID '20180122053037':
        status: CA_UNREACHABLE
        ca-error: Server at https://idmipa01.nix.mds.xyz/ipa/xml failed request, will retry: 4016 (RPC failed at server.  Failed to authenticate to CA REST API).

Request ID '20180122053042':
        status: CA_UNREACHABLE
        ca-error: Server at https://idmipa01.nix.mds.xyz/ipa/xml failed request, will retry: 4016 (RPC failed at server.  Failed to authenticate to CA REST API).

not much.  Hmm.  Running a manual resubmit appears to have done something:

# getcert resubmit -i 20180122053042
Resubmitting "20180122053042" to "IPA".

# getcert resubmit -i 20180122053037
Resubmitting "20180122053037" to "IPA".

New dates are posted for the certs, which looks promising:

Request ID '20180122053037':
        status: POST_SAVED_CERT
        stuck: no
        key pair storage: type=NSSDB,location='/etc/dirsrv/slapd-NIX-MDS-XYZ',nickname='Server-Cert',token='NSS Certificate DB',pinfile='/etc/dirsrv/slapd-NIX-MDS-XYZ/pwdfile.txt'
        certificate: type=NSSDB,location='/etc/dirsrv/slapd-NIX-MDS-XYZ',nickname='Server-Cert',token='NSS Certificate DB'
        CA: IPA
        issuer: CN=Certificate Authority,O=NIX.MDS.XYZ
        subject: CN=idmipa01.nix.mds.xyz,O=NIX.MDS.XYZ
        expires: 2024-09-26 05:16:52 UTC
        principal name: ldap/idmipa01.nix.mds.xyz@NIX.MDS.XYZ
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command:
        post-save command: /usr/libexec/ipa/certmonger/restart_dirsrv NIX-MDS-XYZ
        track: yes
        auto-renew: yes
Request ID '20180122053042':
        status: MONITORING
        stuck: no
        key pair storage: type=NSSDB,location='/etc/httpd/alias',nickname='Server-Cert',token='NSS Certificate DB',pinfile='/etc/httpd/alias/pwdfile.txt'
        certificate: type=NSSDB,location='/etc/httpd/alias',nickname='Server-Cert',token='NSS Certificate DB'
        CA: IPA
        issuer: CN=Certificate Authority,O=NIX.MDS.XYZ
        subject: CN=idmipa01.nix.mds.xyz,O=NIX.MDS.XYZ
        expires: 2024-09-26 05:16:38 UTC
        principal name: HTTP/idmipa01.nix.mds.xyz@NIX.MDS.XYZ
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command:
        post-save command: /usr/libexec/ipa/certmonger/restart_httpd
        track: yes
        auto-renew: yes

Let's check the final result but before that, let's check the date.  It seems odd that it picked 09-26 above but I don't care as long as it works properly:

# date
Mon Sep 26 01:18:19 EDT 2022

Seems ipactl restart or start did the date change hence the date of 09-26.  Let's check the certs now:

# getcert list|grep -Ei "Request ID|status:|stuck:|expires"
Request ID '20180122053031':
        status: MONITORING
        stuck: no
        expires: 2024-09-15 05:15:58 UTC
Request ID '20180122053032':
        status: MONITORING
        stuck: no
        expires: 2024-09-15 05:09:34 UTC
Request ID '20180122053033':
        status: MONITORING
        stuck: no
        expires: 2024-09-15 05:14:47 UTC
Request ID '20180122053034':
        status: MONITORING
        stuck: no
        expires: 2042-09-11 09:07:22 UTC
Request ID '20180122053035':
        status: MONITORING
        stuck: no
        expires: 2024-08-31 09:03:44 UTC
Request ID '20180122053036':
        status: MONITORING
        stuck: no
        expires: 2024-08-31 09:03:43 UTC
Request ID '20180122053037':
        status: MONITORING
        stuck: no
        expires: 2024-09-26 05:16:52 UTC
Request ID '20180122053042':
        status: MONITORING
        stuck: no
        expires: 2024-09-26 05:16:38 UTC
Request ID '20180122053135':
        status: MONITORING
        stuck: no
        expires: 2023-09-26 00:54:45 UTC

And we are done.  Seems our certs are all renewed now and our IDMIPA host is back to a working state.  At least idmipa01 is!  Let's fix the replica:

[idmipa01] # ipa-replica-manage list -v
idmipa01.nix.mds.xyz: master
idmipa02.nix.mds.xyz: master

[idmipa02 ] # ipa-replica-manage list -v
idmipa02.nix.mds.xyz

idmipa01.nix.mds.xyz: replica
  last update status: Error (18) Replication error acquiring replica: Incremental update transient warning.  Backing off, will retry update later. (transient warning)
  last update ended: 1970-01-01 00:00:00+00:00

using this command:

# ipa-replica-manage re-initialize –from idmipa01.nix.mds.xyz

But alas, no, it's master/master setup:

# ipa-replica-manage re-initialize –from idmipa02.nix.mds.xyz
'idmipa02.nix.mds.xyz' has no replication agreement for 'idmipa02.nix.mds.xyz'

[idmipa01]
# sha256sum /var/lib/ipa/replica-info-idmipa02.nix.mds.xyz.gpg
14553b94d6fad6350ce1cf2896c757657d638c8a08b250d13eac6bbacf5d3383  /var/lib/ipa/replica-info-idmipa02.nix.mds.xyz.gpg

[ idmipa02 ]
sha256sum /var/lib/ipa/replica-info-idmipa02.nix.mds.xyz.gpg
14553b94d6fad6350ce1cf2896c757657d638c8a08b250d13eac6bbacf5d3383  /var/lib/ipa/replica-info-idmipa02.nix.mds.xyz.gpg

Reissue the following:

# ipa-replica-install –setup-ca –setup-dns –forwarder=192.168.0.224 /var/lib/ipa/replica-info-idmipa02.nix.mds.xyz.gpg
Your system may be partly configured.
Run /usr/sbin/ipa-server-install –uninstall to clean up.

ipapython.admintool: ERROR    IPA server is already configured on this system.
If you want to reinstall the IPA server, please uninstall it first using 'ipa-server-install –uninstall'.
ipapython.admintool: ERROR    The ipa-replica-install command failed. See /var/log/ipareplica-install.log for more information

However, it complained. Let's try to find another way.  So rebooted instead to see if that will work.  Nothing happened, apparently, though I did not fully check.  However, running the following worked well:

[ idmipa02 ]

# ipa-replica-manage re-initialize –from idmipa01.nix.mds.xyz
Directory Manager password:

Update in progress, 4 seconds elapsed
Update succeeded

# getcert list|grep -Ei "Request ID|status:|stuck:|expires"
Request ID '20180122053638':
        status: MONITORING
        stuck: no
        expires: 2024-09-15 05:15:58 UTC
Request ID '20180122053639':
        status: MONITORING
        stuck: no
        expires: 2024-09-15 05:09:34 UTC
Request ID '20180122053640':
        status: MONITORING
        stuck: no
        expires: 2024-09-15 05:14:47 UTC
Request ID '20180122053641':
        status: MONITORING
        stuck: no
        expires: 2036-11-21 07:32:02 UTC
Request ID '20180122053642':
        status: MONITORING
        stuck: no
        expires: 2024-08-31 09:03:44 UTC
Request ID '20180122053643':
        status: NEED_TO_SUBMIT
        stuck: no
        expires: 2022-08-27 17:23:10 UTC
Request ID '20180122053644':
        status: CA_UNREACHABLE
        stuck: no
        expires: 2022-09-29 17:22:58 UTC
Request ID '20180122053649':
        status: CA_UNREACHABLE
        stuck: no
        expires: 2022-09-29 17:22:45 UTC
Request ID '20180122053742':
        status: MONITORING
        stuck: no
        expires: 2023-09-26 00:54:54 UTC

And a full restart went perfectly well:

# ipactl restart
Restarting Directory Service
Restarting krb5kdc Service
Restarting kadmin Service
Restarting named Service
Restarting httpd Service
Restarting ipa-custodia Service
Restarting ntpd Service
Restarting pki-tomcatd Service
Restarting smb Service
Restarting winbind Service
Restarting ipa-otpd Service
Restarting ipa-dnskeysyncd Service
ipa: INFO: The ipactl command was successful

But giving the above a few moments, certs still didn't update after some time.  Trying to run the following:

# ipa-certupdate

This got me further but one is still unreachable with error:

# getcert list|grep -Ei "Request ID|status:|stuck:|expires"
Request ID '20180122053638':
        status: MONITORING
        stuck: no
        expires: 2024-09-15 05:15:58 UTC
Request ID '20180122053639':
        status: MONITORING
        stuck: no
        expires: 2024-09-15 05:09:34 UTC
Request ID '20180122053640':
        status: MONITORING
        stuck: no
        expires: 2024-09-15 05:14:47 UTC
Request ID '20180122053641':
        status: MONITORING
        stuck: no
        expires: 2042-09-11 09:07:22 UTC
Request ID '20180122053642':
        status: MONITORING
        stuck: no
        expires: 2024-08-31 09:03:44 UTC
Request ID '20180122053643':
        status: CA_UNREACHABLE

        stuck: no
        expires: 2022-08-27 17:23:10 UTC
Request ID '20180122053644':
        status: MONITORING
        stuck: no
        expires: 2024-09-27 23:42:10 UTC
Request ID '20180122053649':
        status: MONITORING
        stuck: no
        expires: 2024-09-27 23:41:58 UTC
Request ID '20180122053742':
        status: MONITORING
        stuck: no
        expires: 2023-09-26 00:54:54 UTC

A more detailed look from getcert list :

Request ID '20180122053643':
        status: CA_UNREACHABLE
        ca-error: Error 60 connecting to https://idmipa02.nix.mds.xyz:8443/ca/agent/ca/profileReview: Peer certificate cannot be authenticated with given CA certificates.
        stuck: no

Resubmit did nothing:

# getcert resubmit -i 20180122053643

Checking the CA we receive:

# ipa ca-show ipa  -v
Usage: ipa [global-options] ca-show NAME [options]

# ipa ca-show ipa
ipa: ERROR: Failed to authenticate to CA REST API

Digging into the getcert list and /var/log/pki/pki-tomcat/ca/debug logs further, we get the following messages:

# getcert list
ca-error: Error 60 connecting to https://idmipa02.nix.mds.xyz:8443/ca/agent/ca/profileReview: Peer certificate cannot be authenticated with given CA certificates.

# /var/log/pki/pki-tomcat/ca/debug

[28/Sep/2022:00:59:32][localhost-startStop-1]: CertUtils: verifySystemCertsByTag() failed: java.lang.Exception: Certutils.verifySystemCertValidityByNickname:  faliled: nickname: Server-Cert cert-pki-cacause: java.lang.Exception: Certutils.verifySystemCertValidityByNickname:  failed: nickname: Server-Cert cert-pki-ca
[28/Sep/2022:00:59:32][localhost-startStop-1]: SignedAuditLogger: event CIMC_CERT_VERIFICATION
[28/Sep/2022:00:59:32][localhost-startStop-1]: SignedAuditLogger: event CIMC_CERT_VERIFICATION
java.lang.Exception: Certutils.verifySystemCertValidityByNickname:  faliled: nickname: Server-Cert cert-pki-cacause: java.lang.Exception: Certutils.verifySystemCertValidityByNickname:  failed: nickname: Server-Cert cert-pki-ca

[28/Sep/2022:00:59:32][localhost-startStop-1]: SelfTestSubsystem: Shutting down server due to selftest failure: Certutils.verifySystemCertValidityByNickname:  faliled: nickname: Server-Cert cert-pki-cacause: java.lang.Exception: Certutils.verifySystemCertValidityByNickname:  failed: nickname: Server-Cert cert-pki-ca
java.lang.Exception: Certutils.verifySystemCertValidityByNickname:  faliled: nickname: Server-Cert cert-pki-cacause: java.lang.Exception: Certutils.verifySystemCertValidityByNickname:  failed: nickname: Server-Cert cert-pki-ca

[28/Sep/2022:00:59:33][http-bio-8080-exec-1]: Failed to read product version String. java.io.FileNotFoundException: /usr/share/pki/CS_SERVER_VERSION (No such file or directory)

Which gives us a lead, but nothing came of that error in reading and searching.  Then paid more attention and see this:

Request ID '20180122053643':
        status: CA_UNREACHABLE
        stuck: no
        expires: 2022-08-27 17:23:10 UTC
        

Cert's expired.  Time to roll back the clock:

# ipactl restart
Restarting Directory Service
Restarting krb5kdc Service
Restarting kadmin Service
Restarting named Service
Restarting httpd Service
Failed to restart httpd Service
Shutting down
Hint: You can use –ignore-service-failure option for forced start in case that a non-critical service failed
Aborting ipactl

but nope this fails.  Let's try the fix command:

# ipa-cert-fix

                          WARNING

ipa-cert-fix is intended for recovery when expired certificates
prevent the normal operation of IPA.  It should ONLY be used
in such scenarios, and backup of the system, especially certificates
and keys, is STRONGLY RECOMMENDED.


The following certificates will be renewed:

Dogtag sslserver certificate:
  Subject: CN=idmipa02.nix.mds.xyz,O=NIX.MDS.XYZ
  Serial:  268369924
  Expires: 2022-08-27 17:23:10

Enter "yes" to proceed: yes
Proceeding.
Command 'pki-server cert-fix –ldapi-socket /var/run/slapd-NIX-MDS-XYZ.socket –agent-uid ipara –cert sslserver' returned non-zero exit status 1
The ipa-cert-fix command failed.

Getting the typical CSR error:

# pki-server cert-fix –ldapi-socket /var/run/slapd-NIX-MDS-XYZ.socket –agent-uid ipara –cert sslserver
INFO: Loading password config: /etc/pki/pki-tomcat/password.conf
INFO: Fixing the following system certs: [‘sslserver’]
INFO: Renewing the following additional certs: []
SASL/EXTERNAL authentication started
SASL username: gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth
SASL SSF: 0
INFO: Stopping the instance to proceed with system cert renewal
INFO: Configuring LDAP password authentication
INFO: Setting pkidbuser password via ldappasswd
SASL/EXTERNAL authentication started
SASL username: gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth
SASL SSF: 0
INFO: Selftests disabled for subsystems: ca
INFO: Resetting password for uid=ipara,ou=people,o=ipaca
SASL/EXTERNAL authentication started
SASL username: gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth
SASL SSF: 0
INFO: Creating a temporary sslserver cert
INFO: Getting sslserver cert info for ca
INFO: Trying to create a new temp cert for sslserver.
INFO: Generate temp SSL certificate
INFO: Getting sslserver cert info for ca
INFO: Selftests enabled for subsystems: ca
INFO: Restoring previous LDAP configuration
ERROR: Unable to find CSR for sslserver cert

Let's get the CSR:

# grep -Ei "csr=" -A19 /var/lib/certmonger/requests/20180122053643 | grep -Evi "CATE REQ" | tr -d '[:space:]'
MIIDNzCCA…………………………………………….4gpgJAb+hM=

Check that you added the entry correctly:

# ca.sslserver.certreq=MIIDNzCCAh8………………………………………………………………………….gpgJAb+hM=

Try the IPA fix once more.  This time we have success:

# ipa-cert-fix

                          WARNING

ipa-cert-fix is intended for recovery when expired certificates
prevent the normal operation of IPA.  It should ONLY be used
in such scenarios, and backup of the system, especially certificates
and keys, is STRONGLY RECOMMENDED.


The following certificates will be renewed:

Dogtag sslserver certificate:
  Subject: CN=idmipa02.nix.mds.xyz,O=NIX.MDS.XYZ
  Serial:  268369924
  Expires: 2022-08-27 17:23:10

Enter "yes" to proceed: yes
Proceeding.
Renewed Dogtag sslserver certificate:
  Subject: CN=idmipa02.nix.mds.xyz,O=NIX.MDS.XYZ
  Serial:  268369929
  Expires: 2024-09-17 05:32:43

The ipa-cert-fix command was successful, apparently.  Restarting services to confirm:

[idmipa02] # ipactl status
Directory Service: RUNNING
krb5kdc Service: RUNNING
kadmin Service: RUNNING
named Service: RUNNING
httpd Service: RUNNING
ipa-custodia Service: RUNNING
ntpd Service: RUNNING
pki-tomcatd Service: RUNNING
smb Service: RUNNING
winbind Service: RUNNING
ipa-otpd Service: RUNNING
ipa-dnskeysyncd Service: RUNNING
ipa: INFO: The ipactl command was successful

[idmipa02] # getcert list|grep -Ei "Request ID|status:|stuck:|expires"
Request ID '20180122053638':
        status: MONITORING
        stuck: no
        expires: 2024-09-15 05:15:58 UTC
Request ID '20180122053639':
        status: MONITORING
        stuck: no
        expires: 2024-09-15 05:09:34 UTC
Request ID '20180122053640':
        status: MONITORING
        stuck: no
        expires: 2024-09-15 05:14:47 UTC
Request ID '20180122053641':
        status: MONITORING
        stuck: no
        expires: 2042-09-11 09:07:22 UTC
Request ID '20180122053642':
        status: MONITORING
        stuck: no
        expires: 2024-08-31 09:03:44 UTC
Request ID '20180122053643':
        status: MONITORING
        stuck: no
        expires: 2024-09-15 05:46:41 UTC
Request ID '20180122053644':
        status: MONITORING
        stuck: no
        expires: 2024-09-27 23:42:10 UTC
Request ID '20180122053649':
        status: MONITORING
        stuck: no
        expires: 2024-09-27 23:41:58 UTC
Request ID '20180122053742':
        status: MONITORING
        stuck: no
        expires: 2023-09-26 00:54:54 UTC

And just to be sure:

[idmipa01]
ipa config-show | grep 'IPA CA renewal master'
  IPA CA renewal master: idmipa01.nix.mds.xyz
  
[idmipa02]
ipa config-show | grep 'IPA CA renewal master'
  IPA CA renewal master: idmipa01.nix.mds.xyz
 

Hope this helps someone!

Cheers,
Tom

REFERENCES:
https://lists.fedoraproject.org/archives/list/freeipa-users@lists.fedorahosted.org/thread/JYQU7PJGY4QV7C6S34Q7VOAAGU7FGLWF/
https://floblanc.wordpress.com/2017/09/11/troubleshooting-freeipa-pki-tomcatd-fails-to-start/
https://access.redhat.com/solutions/3081821
https://access.redhat.com/articles/4062581
https://access.redhat.com/solutions/3357261
https://rcritten.wordpress.com/2017/09/20/peer-certificate-cannot-be-authenticated-with-given-ca-certificates/
https://lists.fedorahosted.org/archives/list/freeipa-users@lists.fedorahosted.org/thread/XSMWWPJU2VRUIGE6SRAHYAJF7BYBCNOE/
https://frasertweedale.github.io/blog-redhat/posts/2019-05-24-ipa-cert-fix.html
https://lists.fedorahosted.org/archives/list/freeipa-users@lists.fedorahosted.org/thread/P73XKHFUJ75VHOJWK2A6ZTLZQ7I2IYE6/
https://floblanc.wordpress.com/2016/12/19/troubleshooting-certmonger-issues-with-freeipa/
https://access.redhat.com/solutions/4908451
https://access.redhat.com/solutions/4852721
https://floblanc.wordpress.com/2016/12/19/troubleshooting-certmonger-issues-with-freeipa/
https://floblanc.wordpress.com/2016/12/19/troubleshooting-certmonger-issues-with-freeipa/
https://serverfault.com/questions/709470/date-and-hwclock-not-in-sync-why

vmware: no healthy upstream

After a hard reset, a greeting of:

no healthy upstream

pops up from the vSphere Client.  Login as root and issue:

service-control –start vmware-vpxd

to see if there's any  additional information in regards to this error.  You may  or may not receive more info.  Check the time and NTP settings. There's a good chance time is not synced up.

date

Login to the management console.  For example:

https://vcsa01.nix.mds.xyz:5480/#/login

If it fails to login with:

Unable to login

check space with:

# df -h |grep 100
/dev/mapper/log_vg-log                    9.8G  9.5G     0 100% /storage/log

Clear space by removing old log files, for example:

root@vcsa01 [ ~ ]# df -h |grep 100
/dev/mapper/log_vg-log                    9.8G  9.5G     0 100% /storage/log
root@vcsa01 [ ~ ]# cd /storage/log
root@vcsa01 [ /storage/log ]# du -sh *|grep G
9.4G    vmware
root@vcsa01 [ /storage/log ]# cd vmware/
root@vcsa01 [ /storage/log/vmware ]# du -sh *|grep G
1.6G    eam
2.7G    lookupsvc
root@vcsa01 [ /storage/log/vmware ]# cd lookupsvc/
root@vcsa01 [ /storage/log/vmware/lookupsvc ]# du -sh *|grep G
2.6G    tomcat
root@vcsa01 [ /storage/log/vmware/lookupsvc ]# cd tomcat/
root@vcsa01 [ /storage/log/vmware/lookupsvc/tomcat ]#
root@vcsa01 [ /storage/log/vmware/lookupsvc/tomcat ]# rm -rf localhost_access.2021*
root@vcsa01 [ /storage/log/vmware/lookupsvc/tomcat ]# df -h .
Filesystem              Size  Used Avail Use% Mounted on
/dev/mapper/log_vg-log  9.8G  8.0G  1.4G  86% /storage/log
root@vcsa01 [ /storage/log/vmware/lookupsvc/tomcat ]#

After cleaning up the space in a few more folders, reboot the appliance:

reboot

You should now be able to login after the space is freed.  Try to loign again to the management console (Port 5480) to disable and enable Time Synchronization if UI still doesn't show up.  If not, issue:

root@vcsa01 [ ~ ]# service-control –start applmgmt

From the vSphere Client SSH session.  If you get a certificate expiration failure:

Exception in invoking authentication handler [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1076)

Renew the self signed cert, or official certificate.  Use this page:

https://kb.vmware.com/s/article/76719

Example output:

root@vcsa01 [ /tmp ]# ./fixsts.sh
NOTE: This works on external and embedded PSCs
This script will do the following
1: Regenerate STS certificate
What is needed?
1: Offline snapshots of VCs/PSCs
2: SSO Admin Password
IMPORTANT: This script should only be run on a single PSC per SSO domain
==================================
Resetting STS certificate for vcsa01.nix.mds.xyz started on Mon Aug  1 04:23:46 UTC 2022


Detected DN: cn=vcsa01.nix.mds.xyz,ou=Domain Controllers,dc=vsphere,dc=local
Detected PNID: vcsa01.nix.mds.xyz
Detected PSC: vcsa01.nix.mds.xyz
Detected SSO domain name: vsphere.local
Detected Machine ID: 310ae9cb-82a9-4fa4-bcd4-d34b054d0090
Detected IP Address: 192.168.0.33
Domain CN: dc=vsphere,dc=local
==================================
==================================

Detected Root's certificate expiration date: 2030 Jun 3
Detected today's date: 2022 Aug 1
==================================

Exporting and generating STS certificate

Status : Success
Using config file : /tmp/vmware-fixsts/certool.cfg
Status : Success


Enter password for administrator@vsphere.local:
Highest tenant credentials index : 1
Exporting tenant 1 to /tmp/vmware-fixsts

Deleting tenant 1

Highest trusted cert chains index: 1
Exporting trustedcertchain 1 to /tmp/vmware-fixsts

Deleting trustedcertchain 1

Applying newly generated STS certificate to SSO domain
adding new entry "cn=TenantCredential-1,cn=vsphere.local,cn=Tenants,cn=IdentityManager,cn=Services,dc=vsphere,dc=local"

adding new entry "cn=TrustedCertChain-1,cn=TrustedCertificateChains,cn=vsphere.local,cn=Tenants,cn=IdentityManager,cn=Services,dc=vsphere,dc=local"


Replacement finished – Please restart services on all vCenters and PSCs in your SSO domain
==================================
IMPORTANT: In case you're using HLM (Hybrid Linked Mode) without a gateway, you would need to re-sync the certs from Cloud to On-Prem after following this procedure
==================================
==================================
root@vcsa01 [ /tmp ]#

Try to login again.  

Cheers,
Tom

NTPD: Setting up an NTP server on DD-WRT or OpenWRT

Recent power outages and ISP outages left my network without a proper internal NTP server which was, coincidently, installed on an ESXi host.  Having to revert to an external NTP server for the time being, a recent outage with my ISP highlighted the fact that even that isn't enough.  The ISP outage () made it clear I need a solution that is:

  1. Sitting on a lower power device and external to my LAB, so isolated from any large server hosted device.
  2. Doesn't depend on DNS to syn cup time in case an ISP is offline.  
  3. Maintain an accurate time on it's own so it itself will be a reliable source of time when everything is offline.

So went with an OpenWRT and a Raspberry Pi device for just this very thing.  This is super simple:

opkg update
opkg install ntpd
/etc/init.d/sysntpd disable
/etc/init.d/ntpd enable
/etc/init.d/ntpd start
netstat -l | grep ntp

Configure the external NTP servers to use ( NOTE: See additional information below in the EDIT section for extra iptions or the configuration will not work. ):

root@OWRT01:~# cat /etc/config/system

config system
        option ttylogin '0'
        option log_size '64'
        option urandom_seed '0'
        option hostname 'OWRT01'
        option log_proto 'udp'
        option conloglevel '8'
        option cronloglevel '5'
        option timezone 'EST5EDT,M3.2.0,M11.1.0'
        option zonename 'America/Toronto'
        option log_ip 192.168.0.14
        option log_port 514
        option log_proto udp


config timeserver 'ntp'
        list server '0.ca.pool.ntp.org'
        list server '1.ca.pool.ntp.org'
        list server '2.ca.pool.ntp.org'
root@OWRT01:~#

Set the time manually, in the event the system can't sync it's time with an external server:

# date
# date -k

So our brand new NTP server is sitting on:

192.168.0.12

Let's now set the Date / Time to sync from this NTP server.  For Cisco switches:

mdscisco01#conf t
Enter configuration commands, one per line.  End with CNTL/Z.
mdscisco01(config)#ntp server 192.168.0.12
mdscisco01(config)#end
mdscisco01#show run
mdscisco01#show running-config

Ensure local time is also set correctly:

mdscisco02#
mdscisco02#conf t
Enter configuration commands, one per line.  End with CNTL/Z.
mdscisco02(config)#clock timezone EST -5
mdscisco02(config)#end
mdscisco02#clock set 11:52:00 July 10 2022
mdscisco02#copy run
mdscisco02#copy running-config startup-config
Destination filename [startup-config]?
Building configuration…
Compressed configuration from 7043 bytes to 2639 bytes[OK]
mdscisco02#

For Linux Servers:

[root@mbpc-pc ~]# cat /etc/ntp.conf|grep -Eiv "^#"
driftfile /var/lib/ntp/drift
restrict default kod nomodify notrap nopeer noquery
restrict -6 default kod nomodify notrap nopeer noquery
restrict 127.0.0.1
restrict -6 ::1
server 192.168.0.12
server idmipa01.nix.mds.xyz prefer
server idmipa02.nix.mds.xyz prefer
server 0.rhel.pool.ntp.org iburst
server 1.rhel.pool.ntp.org iburst
server 2.rhel.pool.ntp.org iburst
server 3.rhel.pool.ntp.org iburst
includefile /etc/ntp/crypto/pw
keys /etc/ntp/keys

For DD-WRT, configure via the basic config to use the OpenWRT NTP server:

https://i2.wp.com/www.microdevsys.com/WordPressImages/DD-WRT-NTP-Setup.PNG?ssl=1

Configure ESXi hosts you may have:

https://i0.wp.com/www.microdevsys.com/wordpressimages/DD-WRT-NTP-Setup-ESXi-Config.PNG?ssl=1

REF: https://openwrt.org/docs/guide-user/services/ntp/client-server 
REF: https://www.cisco.com/c/en/us/td/docs/switches/lan/catalyst3750x_3560x/software/release/12-2_55_se/configuration/guide/3750xscg/swadmin.html 
REF: https://wiki.dd-wrt.com/wiki/index.php/Network_Time_Protocol#:~:text=You%20cannot%20set%20your%20time,to%20match%20your%20local%20time.

EDIT: Aug 17 2022

An additional option is required to make this a server:

root@OWRT01:/tmp/log# cat /etc/config/system

config system
        option ttylogin '0'
        option log_size '64'
        option urandom_seed '0'
        option hostname 'OWRT01'
        option log_proto 'udp'
        option conloglevel '8'
        option cronloglevel '5'
        option timezone 'EST5EDT,M3.2.0,M11.1.0'
        option zonename 'America/Toronto'
        option log_ip 192.168.0.14
        option log_port 514
        option log_proto udp


config timeserver 'ntp'
        list server '0.ca.pool.ntp.org'
        list server '1.ca.pool.ntp.org'
        list server '2.ca.pool.ntp.org'

        option enable_server '1'
root@OWRT01:/tmp/log#

Restart the service:

root@OWRT01:/tmp/log# service ntpd restart

The above results in:

root@OWRT01:/tmp/log# cat /var/etc/ntpd.conf
driftfile /var/lib/ntp/ntp.drift

restrict default limited kod nomodify notrap nopeer
restrict -6 default limited kod nomodify notrap nopeer
restrict source noquery

# No limits for local monitoring
restrict 127.0.0.1
restrict -6 ::1

server 0.ca.pool.ntp.org iburst
server 1.ca.pool.ntp.org iburst
server 2.ca.pool.ntp.org iburst
root@OWRT01:/tmp/log#

Without the above, errors such as these will be shown:

Jan  1 03:51:00 DD-WRT-BEESWAX user.info : [ntpclient] : Network Time Protocol client trying to stop
Jan  1 03:51:00 DD-WRT-BEESWAX daemon.info process_monitor[1824]: _evalpid:ntpclient 192.168.0.12
Jan  1 03:51:00 DD-WRT-BEESWAX daemon.debug ntpclient[7036]: Connecting to 192.168.0.12 [192.168.0.12] …
Jan  1 03:51:03 DD-WRT-BEESWAX daemon.debug ntpclient[7036]: Timed out waiting for 192.168.0.12 [192.168.0.12].
Jan  1 03:51:03 DD-WRT-BEESWAX daemon.err process_monitor[1824]: cyclic NTP Update failed (servers 192.168.0.12)
Jan  1 03:51:33 DD-WRT-BEESWAX daemon.info process_monitor[1824]: _evalpid:stopservice ntpc -f
Jan  1 03:51:33 DD-WRT-BEESWAX user.info : _evalpid:/sbin/service ntpc stop

Without the server line, the NTP server will receive the request but will refuse to send the client time.  A few typical messages:

root@OWRT01:~# strace -s 256 -f -p 701
[pid   701] _newselect(59, [48 49 50 51 53 54 55 56 57 58], NULL, NULL, NULL) = 1 (in [54])
[pid   701] clock_gettime(CLOCK_REALTIME, {tv_sec=1660757832, tv_nsec=325460418}) = 0
[pid   701] recvmsg(54, {msg_name={sa_family=AF_INET, sin_port=htons(34957), sin_addr=inet_addr(“192.168.0.21″)}, msg_namelen=28->16, msg_iov=[{iov_base=”\33\0\4\372\0\1\0                 \0\0\1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\203\252\360=\246>\215\323”, iov_len=2120}], msg_iovlen=1, msg_control=[{cmsg_len=20, cmsg_level=SOL_SOCKET, cmsg_type=SCM_TIMESTAMPNS, cmsg_data={tv_sec=1660757832, tv_nsec=324808641}}], msg_controllen=20, msg_flags=0}, 0) = 48
[pid   701] recvmsg(54, {msg_namelen=28}, 0) = -1 EAGAIN (Resource temporarily unavailable)
[pid   701] _newselect(59, [48 49 50 51 53 54 55 56 57 58], NULL, NULL, NULL) = 1 (in [54])
[pid   701] clock_gettime(CLOCK_REALTIME, {tv_sec=1660757833, tv_nsec=286244526}) = 0
[pid   701] recvmsg(54, {msg_name={sa_family=AF_INET, sin_port=htons(44107), sin_addr=inet_addr(“192.168.0.19″)}, msg_namelen=28->16, msg_iov=[{iov_base=”\33\0\4\372\0\1\0                 \0\0\1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\203\252\202\264\200\n\215\34”, iov_len=2120}], msg_iovlen=1, msg_control=[{cmsg_len=20, cmsg_level=SOL_SOCKET, cmsg_type=SCM_TIMESTAMPNS, cmsg_data={tv_sec=1660757833, tv_nsec=285534936}}], msg_controllen=20, msg_flags=0}, 0) = 48
[pid   701] recvmsg(54, {msg_namelen=28}, 0) = -1 EAGAIN (Resource temporarily unavailable)

REFhttps://oldwiki.archive.openwrt.org/doc/howto/ntp.client
REFhttps://forum.openwrt.org/t/ntp-server-through-busybox-ntp/91344/6

HTH


 


     
  Copyright © 2003 - 2013 Tom Kacperski (microdevsys.com). All rights reserved.

Creative Commons License
This work is licensed under a Creative Commons Attribution 3.0 Unported License