So /etc/resolv.conf was getting consistently overwritten even though NetworkManager was removed using rpm -e $(rpm -aq|grep Network Manager).  To find the culprit, we checked what is running on this CentOS 7 system using:

[root@idmipa01 log]# systemctl list-units –type service|grep -i network
network.service                    loaded active exited  LSB: Bring up/down networking
ntpd.service                       loaded active running Network Time Service
rhel-domainname.service            loaded active exited  Read and set NIS domainname from /etc/sysconfig/network
rhel-import-state.service          loaded active exited  Import network configuration from initramfs
[root@idmipa01 log]# systemctl list-units –type service –all|grep -i network
  network.service                        loaded    active   exited  LSB: Bring up/down networking
â NetworkManager-wait-online.service     not-found inactive dead    NetworkManager-wait-online.service
â NetworkManager.service                 not-found inactive dead    NetworkManager.service
  ntpd.service                           loaded    active   running Network Time Service
  rhel-domainname.service                loaded    active   exited  Read and set NIS domainname from /etc/sysconfig/network
  rhel-import-state.service              loaded    active   exited  Import network configuration from initramfs
[root@idmipa01 log]# systemctl disable NetworkManager.service
Failed to execute operation: Access denied
[root@idmipa01 log]# grep USER_AVC /var/log/audit/audit.log | tail -n1 | audit2why
type=USER_AVC msg=audit(1479956039.484:422): pid=1 uid=0 auid=4294967295 ses=4294967295 subj=system_u:system_r:init_t:s0 msg='avc:  denied  { disable } for auid=0 uid=0 gid=0 cmdline="systemctl disable NetworkManager.service" scontext=unconfined_u:unconfined_r:unconfined_t:s0-s0:c0.c1023 tcontext=system_u:system_r:init_t:s0 tclass=service  exe="/usr/lib/systemd/systemd" sauid=0 hostname=? addr=? terminal=?'

        Was caused by:
                Missing type enforcement (TE) allow rule.

                You can use audit2allow to generate a loadable module to allow this access.

[root@idmipa01 log]# grep USER_AVC /var/log/audit/audit.log | tail -n1 | audit2allow -M systemd-allow
******************** IMPORTANT ***********************
To make this policy package active, execute:

semodule -i systemd-allow.pp

[root@idmipa01 log]# strings systemd-allow.pp
SE Linux Module
systemd-allow
1.0@
service
disable
object_r@
init_t
unconfined_t
service
object_r
init_t
unconfined_t
[root@idmipa01 log]# semodule -i systemd-allow.pp
[root@idmipa01 log]# systemctl disable NetworkManager.service
[root@idmipa01 log]# systemctl disable NetworkManager-wait-online.service
[root@idmipa01 log]#
[root@idmipa01 log]# systemctl list-units –type service|grep -i network
network.service                    loaded active exited  LSB: Bring up/down networking
ntpd.service                       loaded active running Network Time Service
rhel-domainname.service            loaded active exited  Read and set NIS domainname from /etc/sysconfig/network
rhel-import-state.service          loaded active exited  Import network configuration from initramfs
[root@idmipa01 log]#
[root@idmipa01 systemd]# grep -iR NetworkManager*
system/basic.target.wants/firewalld.service:Before=NetworkManager.service
system/dbus-org.fedoraproject.FirewallD1.service:Before=NetworkManager.service
[root@idmipa01 systemd]# vi system/basic.target.wants/firewalld.service
[root@idmipa01 systemd]# vi system/dbus-org.fedoraproject.FirewallD1.service
[root@idmipa01 systemd]# ls -altri system/basic.target.wants/firewalld.service
135366996 lrwxrwxrwx. 1 root root 41 Nov 20 22:39 system/basic.target.wants/firewalld.service -> /usr/lib/systemd/system/firewalld.service
[root@idmipa01 systemd]# ls -altri system/dbus-org.fedoraproject.FirewallD1.service
202641569 lrwxrwxrwx. 1 root root 41 Nov 20 22:39 system/dbus-org.fedoraproject.FirewallD1.service -> /usr/lib/systemd/system/firewalld.service
[root@idmipa01 systemd]#

If the service still fails and shows an entry it's because of this:

[root@idmipa01 system]# systemctl show NetworkManager.service –property=Id,Names,Description
Id=NetworkManager.service
Names=NetworkManager.service
Description=NetworkManager.service
[root@idmipa01 system]#

Unfortunatley no idea how to remove those properties.  🙁

Let's add a watcher and see:

[root@idmipa01 audit]# /sbin/auditctl -w /etc/resolv.conf -p war -k /root/resolv.conf-file
[root@idmipa01 audit]# chattr -i /etc/resolv.conf
[root@idmipa01 audit]# lsattr /etc/resolv.conf
—————- /etc/resolv.conf
[root@idmipa01 audit]#
[root@idmipa01 audit]# /sbin/ausearch -f /etc/resolv.conf

And sure enough we get that postfix is changing it:

—-
time->Wed Nov 23 23:14:47 2016
type=PATH msg=audit(1479960887.978:293): item=0 name="/etc/resolv.conf" inode=135699633 dev=fd:00 mode=0100644 ouid=0 ogid=0 rdev=00:00 obj=system_u:object_r:net_conf_t:s0 objtype=NORMAL
type=CWD msg=audit(1479960887.978:293):  cwd="/"
type=SYSCALL msg=audit(1479960887.978:293): arch=c000003e syscall=2 success=yes exit=4 a0=7ffb36b6f43a a1=80000 a2=1b6 a3=24 items=1 ppid=1 pid=5527 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm="postfix" exe="/usr/sbin/postfix" subj=system_u:system_r:postfix_master_t:s0 key="/root/resolv.conf-file"
—-
time->Wed Nov 23 23:14:48 2016
type=PATH msg=audit(1479960888.013:301): item=0 name="/etc/resolv.conf" inode=135699633 dev=fd:00 mode=0100644 ouid=0 ogid=0 rdev=00:00 obj=system_u:object_r:net_conf_t:s0 objtype=NORMAL
type=CWD msg=audit(1479960888.013:301):  cwd="/var/spool/postfix"
type=SYSCALL msg=audit(1479960888.013:301): arch=c000003e syscall=2 success=yes exit=3 a0=7f32c163043a a1=80000 a2=1b6 a3=24 items=1 ppid=5545 pid=5546 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm="postconf" exe="/usr/sbin/postconf" subj=system_u:system_r:postfix_master_t:s0 key="/root/resolv.conf-file"

This in turn is started by:

[root@idmipa01 slapd-NIX-MDS-XYZ]# grep postfix access|tail -n 1
[23/Nov/2016:23:42:04 -0500] conn=34 op=5 SRCH base="cn=accounts,dc=nix,dc=mds,dc=xyz" scope=2 filter="(&(uid=postfix)(objectClass=posixAccount)(&(uidNumber=*)(!(uidNumber=0))))" attrs="objectClass uid userPassword uidNumber gidNumber gecos homeDirectory loginShell krbPrincipalName cn memberOf ipaUniqueID ipaNTSecurityIdentifier modifyTimestamp entryusn shadowLastChange shadowMin shadowMax shadowWarning shadowInactive shadowExpire shadowFlag krbLastPwdChange krbPasswordExpiration pwdattribute authorizedService accountexpires useraccountcontrol nsAccountLock host logindisabled loginexpirationtime loginallowedtimemap ipaSshPubKey ipaUserAuthType usercertificate;binary"
[root@idmipa01 slapd-NIX-MDS-XYZ]# pwd
/var/log/dirsrv/slapd-NIX-MDS-XYZ
[root@idmipa01 slapd-NIX-MDS-XYZ]#

[root@idmipa01 slapd-NIX-MDS-XYZ]# ps -ef|grep -i slapd
dirsrv    2206     1  0 23:28 ?        00:00:07 /usr/sbin/ns-slapd -D /etc/dirsrv/slapd-NIX-MDS-XYZ -i /var/run/dirsrv/slapd-NIX-MDS-XYZ.pid -w /var/run/dirsrv/slapd-NIX-MDS-XYZ.startpid
root      2851  2770  0 23:55 pts/1    00:00:00 grep –color=auto -i slapd
[root@idmipa01 slapd-NIX-MDS-XYZ]#

Sep 13 2017:

After trying this again today, finally found where the NetworkManager text comes from.

[root@saltm01 network-scripts]# cat /etc/resolv.conf
# Generated by NetworkManager
search mds.xyz

# No nameservers found; try putting DNS servers into your
# ifcfg files in /etc/sysconfig/network-scripts like so:
#
# DNS1=xxx.xxx.xxx.xxx
# DNS2=xxx.xxx.xxx.xxx
# DOMAIN=lab.foo.com bar.foo.com
nameserver 192.168.0.44
nameserver 192.168.0.45
[root@saltm01 network-scripts]# cd ..
[root@saltm01 sysconfig]# cd ..
[root@saltm01 etc]# grep -iER "Generated by NetworkManager" *
resolv.conf:# Generated by NetworkManager
resolv.conf.save:# Generated by NetworkManager
[root@saltm01 etc]# vi resolv.conf.save
[root@saltm01 etc]# cp -ip resolv.conf.save resolv.conf.save-original
[root@saltm01 etc]# >resolv.conf.save
[root@saltm01 etc]# service network restart
Restarting network (via systemctl):                        [  OK  ]
[root@saltm01 etc]#
[root@saltm01 etc]# cat /etc/resolv.conf

nameserver 192.168.0.44
nameserver 192.168.0.45
[root@saltm01 etc]#
[root@saltm01 etc]#

Better yet, leave the search line in it so do an edit of the above.  🙂

Sep 22 2019:

Problem resurfaced.  This time it was the search string that kept getting reverted.  Sure enough, we have a /etc/resolv.conf.save file as well:

[root@mdskvm-p01 etc]# grep -EiR "# Generated by NetworkManager" *
grep: extlinux.conf: No such file or directory
resolv.conf:# Generated by NetworkManager
resolv.conf.save:# Generated by NetworkManager
[root@mdskvm-p01 etc]#

[root@mdskvm-p01 etc]# grep -Ei search resolv.conf resolv.conf.save
resolv.conf:search mds.xyz nix.mds.xyz mws.mds.xyz
resolv.conf.save:search mds.xyz
[root@mdskvm-p01 etc]#

a host where changes to /etc/resolv.conf were not occurring from had these files:

[root@mdskvm-p02 etc]# grep -EiR "# Generated by NetworkManager" *
grep: extlinux.conf: No such file or directory
resolv.conf:# Generated by NetworkManager
resolv.conf-original:# Generated by NetworkManager
[root@mdskvm-p02 etc]#

Because the second host had no /etc/resolv.conf.save file, the changes were not reverted.  This is because there is a line in the following network script that does the revert:

[root@mdskvm-p02 network-scripts]# vi ifdown-post
#!/bin/sh
# This should be called whenever an interface goes down, not just when
# it is brought down explicitly.
,
,
,
# Remove duplicate DNS entries and shift them,
# to have always correct condition below…
update_DNS_entries

if ! is_false "${PEERDNS}" || is_true "${RESOLV_MODS}" && \
    [ "${DEVICETYPE}" = "ppp" -o "${DEVICETYPE}" = "ippp" -o -n "${DNS1}" \
    -o "${BOOTPROTO}" = "bootp" -o "${BOOTPROTO}" = "dhcp" ] ; then
    if [ -f /etc/resolv.conf.save ]; then
        change_resolv_conf /etc/resolv.conf.save
        rm -f /etc/resolv.conf.save
    fi
    if [ “${DEVICETYPE}” = “ppp” -o “${DEVICETYPE}” = “ippp” ]; then
        if [ -f /etc/ppp/peers/$DEVICE ] ; then
            rm -f /etc/ppp/peers/$DEVICE
        fi
    fi
fi
,
,
,

And change_resolv_conf() effectively copies the contents of /etc/resolv.conf.save right into /etc/resolv.conf thereby overwriting anything in it:

[root@mdskvm-p02 network-scripts]# vi network-functions
# Invoke this when /etc/resolv.conf has changed:
change_resolv_conf ()
{
    s=$(/bin/grep '^[\ \       ]*option' /etc/resolv.conf 2>/dev/null)
    if [ $# -gt 1 ]; then
        if [ “x$s” != “x” ]; then
            s="$s"$'\n'
        fi
        n_args=$#
        while [ $n_args -gt 0 ]; do
            case "$s" in
            *$1*)
                shift
                n_args=$(($n_args-1))
                continue
                ;;
            esac
            s="$s$1"
            shift
            if [ $# -gt 0 ]; then
                s="$s"$'\n'
            fi
            n_args=$(($n_args-1))
        done
    elif [ $# -eq 1 ]; then
        if [ “x$s” != “x” ]; then
            s="$s"$'\n'$(/bin/grep -vF "$s" $1)
        else   
            s=$(cat $1)
        fi      
    fi
    (echo "$s" > /etc/resolv.conf) >/dev/null 2>&1;  
    r=$?
    if [ $r -eq 0 ]; then
        [ -x /sbin/restorecon ] && /sbin/restorecon /etc/resolv.conf >/dev/null 2>&1 # reset the correct context
        /usr/bin/logger -p local7.notice -t "NET" -i "$0 : updated /etc/resolv.conf"
        [ -e /var/run/nscd/socket ] && /usr/sbin/nscd -i hosts # invalidate cache
    fi
    return $r
}

So you have to modify both files in order to persist the changes or simply remove the /etc/resolv.conf.save file.  Happy Searching!  😉 

Cheers,
Tom