Description of problem: ======================= Whenever lookup is performed on the snaps snapd logs are filled with too many "dict OR key (entry-point) is NULL" warning messages. It is unclear why the messages are logged on a successful lookup, also any key which results in NULL and warning is the concern to the user. Either the logs can be improved to suggest what and why the warning is, or if the logs are atmost important than it can be moved to debug mode. Lookup from snaps: ================== [root@wingo .snaps]# pwd /mnt/vol1/etc.1/.snaps [root@wingo .snaps]# ls 1 10 11 12 13 14 15 2 3 4 5 6 7 8 9 [root@wingo .snaps]# cd 9 [root@wingo 9]# ls abrt cron.d fstab kdump.conf motd popt.d rhsm sudoers.d acpi cron.daily gai.conf krb5.conf mtab portreserve rpc sudo-ldap.conf adjtime cron.deny gcrypt ktune.d my.cnf postfix rpm sysconfig aliases cron.hourly ghostscript latrace.conf nanorc ppp rsyslog.conf sysctl.conf aliases.db cron.monthly gnupg latrace.d netconfig prelink.cache rsyslog.d system-release alsa crontab group ld.so.cache NetworkManager prelink.conf rwtab system-release-cpe alternatives cron.weekly group- ld.so.conf networks prelink.conf.d rwtab.d terminfo anacrontab crypttab grub.conf ld.so.conf.d nfsmount.conf printcap samba Trolltech.conf asound.conf csh.cshrc gshadow libaudit.conf nsswitch.conf profile sasl2 tuned.conf at.deny csh.login gshadow- libibverbs.d ntp profile.d scl tune-profiles audisp cups gssapi_mech.conf libreport ntp.conf protocols screenrc udev audit dbus-1 ha.d libuser.conf numad.conf pulse securetty updatedb.conf autofs.conf default hal libvirt oddjob quotagrpadmins security vhostmd autofs_ldap_auth.conf depmod.d host.conf localtime oddjobd.conf quotatab selinux vimrc auto.master dhcp hosts login.defs oddjobd.conf.d rc services virc auto.misc DIR_COLORS hosts.allow logrotate.conf openldap rc0.d sestatus.conf warnquota.conf auto.net DIR_COLORS.256color hosts.deny logrotate.d opt rc1.d setuptool.d wgetrc auto.smb DIR_COLORS.lightbgcolor httpd lsb-release pam.d rc2.d sgml X11 bash_completion.d dracut.conf idmapd.conf lsb-release.d pango rc3.d shadow xdg bashrc dracut.conf.d init ltrace.conf passwd rc4.d shadow- xinetd.d blkid drirc init.d lvm passwd- rc5.d shells xml cas.conf elinks.conf inittab magic pcmcia rc6.d skel yp.conf certmonger environment inputrc mailcap php.d rc.d smartd.conf yum cgconfig.conf ethers ipa mail.rc php.ini rc.local sos.conf yum.conf cgconfig.d event.d iproute2 makedev.d pinforc rc.sysinit ssh yum.repos.d cgrules.conf exports issue man.config pkcs11 readahead.conf ssl cgsnapshot_blacklist.conf favicon.png issue.net maven pki redhat-lsb sssd chkconfig.d filesystems java mcelog plymouth redhat-release statetab cluster fonts jvm mime.types pm request-key.conf statetab.d ConsoleKit foomatic jvm-commmon mke2fs.conf pm-utils-hd-apm-restore.conf request-key.d sudo.conf corosync fprintd.conf kdump-adv-conf modprobe.d polkit-1 resolv.conf sudoers [root@wingo 9]# Logs reported as: ================= [2014-12-05 05:25:06.237052] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 05:25:06.251224] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 05:25:06.256306] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 13:07:46.330055] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 13:07:47.916460] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 13:07:47.944886] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 13:07:47.946733] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 13:07:47.948976] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 13:07:47.951230] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 13:07:47.952526] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 13:07:47.957609] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 13:07:47.973093] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 13:07:47.974922] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 13:07:48.474720] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 13:07:48.489094] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 13:07:48.499566] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 13:07:48.507731] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 13:07:48.511175] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 13:07:48.518885] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 13:07:48.534899] W [dict.c:1352:dict_get_with_ref] (-->/usr/lib64/libglusterfs.so.0(default_lookup_resume+0x12c) [0x7f465a21f3dc] (-->/usr/lib64/glusterfs/3.6.0.36/xlator/features/snapview-server.so(svs_lookup+0x2f3) [0x7f46505a10a3] (-->/usr/lib64/libglusterfs.so.0(dict_get_str_boolean+0x1f) [0x7f465a212cbf]))) 0-dict: dict OR key (entry-point) is NULL [2014-12-05 13:07:48.537708] E [snapview-server-helpers.c:450:__svs_initialise_snapshot_volume] 0-vol1-snapview-server: snap entry for name boot not found [2014-12-05 13:07:48.537753] E [snapview-server.c:219:svs_lookup_snapshot] 0-vol1-snapview-server: failed to create the fs instance for snap boot Version-Release number of selected component (if applicable): ============================================================= glusterfs-3.6.0.36-1.el6rhs.x86_64 How reproducible: ================= always Steps to Reproduce: =================== 1. Create 4 node cluster 2. Create and start a volume (2*2) 3. Mount the volume on client (/mnt/vol1) 4. Create a data from client (cp -rf /etc etc.1) 5. Create snapshot of the volume (9) 6. Activate the snapshot 7. Enable the USS 8. Monitor the logs of snapd (tailf tail -f /var/log/glusterfs/snaps/vol1/snapd.log) 9. cd from client to snapshot (cd /mnt/vol1/etc.1/.snaps/9) 10. ls Actual results: =============== Too many warning messages for "dict OR key (entry-point) is NULL" is reported.
Hi Rajesh, I see this bug listed as a known issue in the known issues tracker bug for 3.0.3. Can you please fill out the doc text after changing the doc type to known issue?
We are getting this warning message for every lookup sent to snapview-server and log file grows very quickly. I think this should go in to 3.0.3
Patch submitted: https://code.engineering.redhat.com/gerrit/#/c/38651/
Verified with build: glusterfs-3.6.0.40-1.el6rhs.x86_64 No logs observed after carrying the steps mentioned in the description. Moving the bug to verified state [root@wingo ~]# cd /mnt/vol_test_fuse/etc.1/.snaps [root@wingo .snaps]# pwd /mnt/vol_test_fuse/etc.1/.snaps [root@wingo .snaps]# cd 9 [root@wingo 9]# ls abrt cron.d fstab kdump.conf motd popt.d rhsm sudoers.d acpi cron.daily gai.conf krb5.conf mtab portreserve rpc sudo-ldap.conf adjtime cron.deny gcrypt ktune.d my.cnf postfix rpm sysconfig aliases cron.hourly ghostscript latrace.conf nanorc ppp rsyslog.conf sysctl.conf aliases.db cron.monthly gnupg latrace.d netconfig prelink.cache rsyslog.d system-release alsa crontab group ld.so.cache NetworkManager prelink.conf rwtab system-release-cpe alternatives cron.weekly group- ld.so.conf networks prelink.conf.d rwtab.d terminfo anacrontab crypttab grub.conf ld.so.conf.d nfsmount.conf printcap samba Trolltech.conf asound.conf csh.cshrc gshadow libaudit.conf nsswitch.conf profile sasl2 tuned.conf at.deny csh.login gshadow- libibverbs.d ntp profile.d scl tune-profiles audisp cups gssapi_mech.conf libreport ntp.conf protocols screenrc udev audit dbus-1 ha.d libuser.conf numad.conf pulse securetty updatedb.conf autofs.conf default hal libvirt oddjob quotagrpadmins security vhostmd autofs_ldap_auth.conf depmod.d host.conf localtime oddjobd.conf quotatab selinux vimrc auto.master dhcp hosts login.defs oddjobd.conf.d rc services virc auto.misc DIR_COLORS hosts.allow logrotate.conf openldap rc0.d sestatus.conf warnquota.conf auto.net DIR_COLORS.256color hosts.deny logrotate.d opt rc1.d setuptool.d wgetrc auto.smb DIR_COLORS.lightbgcolor httpd lsb-release pam.d rc2.d sgml X11 bash_completion.d dracut.conf idmapd.conf lsb-release.d pango rc3.d shadow xdg bashrc dracut.conf.d init ltrace.conf passwd rc4.d shadow- xinetd.d blkid drirc init.d lvm passwd- rc5.d shells xml cas.conf elinks.conf inittab magic pcmcia rc6.d skel yp.conf certmonger environment inputrc mailcap php.d rc.d smartd.conf yum cgconfig.conf ethers ipa mail.rc php.ini rc.local sos.conf yum.conf cgconfig.d event.d iproute2 makedev.d pinforc rc.sysinit ssh yum.repos.d cgrules.conf exports issue man.config pkcs11 readahead.conf ssl cgsnapshot_blacklist.conf favicon.png issue.net maven pki redhat-lsb sssd chkconfig.d filesystems java mcelog plymouth redhat-release statetab cluster fonts jvm mime.types pm request-key.conf statetab.d ConsoleKit foomatic jvm-commmon mke2fs.conf pm-utils-hd-apm-restore.conf request-key.d sudo.conf corosync fprintd.conf kdump-adv-conf modprobe.d polkit-1 resolv.conf sudoers [root@wingo 9]# [root@inception ~]# cat /var/log/glusterfs/snaps/vol_test/snapd.log | grep -i "dict OR key" [root@inception ~]#