Attachment 308995 Details for Bug 450194 – backport of dcache sb patch.

[patch] backport of dcache sb patch.

dcache-lru-rewrite.patch (text/plain), 14.65 KB, created by Josef Bacik on 2008-06-11 20:57:58 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Josef Bacik

Created: 2008-06-11 20:57:58 UTC

Size: 14.65 KB

patch

obsolete

>Index: kernel/fs/dcache.c
>===================================================================
>--- kernel.orig/fs/dcache.c
>+++ kernel/fs/dcache.c
>@@ -60,7 +60,6 @@ static kmem_cache_t *dentry_cache __read
> static unsigned int d_hash_mask __read_mostly;
> static unsigned int d_hash_shift __read_mostly;
> static struct hlist_head *dentry_hashtable __read_mostly;
>-static LIST_HEAD(dentry_unused);
> 
> /* Statistics gathering. */
> struct dentry_stat_t dentry_stat = {
>@@ -116,6 +115,42 @@ static void dentry_iput(struct dentry * 
> 	}
> }
> 
>+/*
>+ * dentry_lru_(add|add_tail|del|del_init) must be called with dcache_lock held.
>+ */
>+static void dentry_lru_add(struct dentry *dentry)
>+{
>+	list_add(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
>+	dentry->d_sb->s_nr_dentry_unused++;
>+	dentry_stat.nr_unused++;
>+}
>+
>+static void dentry_lru_add_tail(struct dentry *dentry)
>+{
>+	list_add_tail(&dentry->d_lru, &dentry->d_sb->s_dentry_lru);
>+	dentry->d_sb->s_nr_dentry_unused++;
>+	dentry_stat.nr_unused++;
>+}
>+
>+static void dentry_lru_del(struct dentry *dentry)
>+{
>+	if (!list_empty(&dentry->d_lru)) {
>+		list_del(&dentry->d_lru);
>+		dentry->d_sb->s_nr_dentry_unused--;
>+		dentry_stat.nr_unused--;
>+	}
>+}
>+
>+static void dentry_lru_del_init(struct dentry *dentry)
>+{
>+	if (!list_empty(&dentry->d_lru)) {
>+		list_del_init(&dentry->d_lru);
>+		dentry->d_sb->s_nr_dentry_unused--;
>+		dentry_stat.nr_unused--;
>+	}
>+}
>+
>+
> /* 
>  * This is dput
>  *
>@@ -175,8 +210,7 @@ repeat:
> 		goto kill_it;
>   	if (list_empty(&dentry->d_lru)) {
>   		dentry->d_flags |= DCACHE_REFERENCED;
>-  		list_add(&dentry->d_lru, &dentry_unused);
>-  		dentry_stat.nr_unused++;
>+		dentry_lru_add(dentry);
>   	}
>  	spin_unlock(&dentry->d_lock);
> 	spin_unlock(&dcache_lock);
>@@ -191,10 +225,7 @@ kill_it: {
> 		/* If dentry was on d_lru list
> 		 * delete it from there
> 		 */
>-  		if (!list_empty(&dentry->d_lru)) {
>-  			list_del(&dentry->d_lru);
>-  			dentry_stat.nr_unused--;
>-  		}
>+		dentry_lru_del(dentry);
>   		list_del(&dentry->d_u.d_child);
> 		dentry_stat.nr_dentry--;	/* For d_free, below */
> 		/*drops the locks, at that point nobody can reach this dentry */
>@@ -270,10 +301,7 @@ int d_invalidate(struct dentry * dentry)
> static inline struct dentry * __dget_locked(struct dentry *dentry)
> {
> 	atomic_inc(&dentry->d_count);
>-	if (!list_empty(&dentry->d_lru)) {
>-		dentry_stat.nr_unused--;
>-		list_del_init(&dentry->d_lru);
>-	}
>+	dentry_lru_del_init(dentry);
> 	return dentry;
> }
> 
>@@ -383,125 +411,160 @@ static void prune_one_dentry(struct dent
> 	spin_lock(&dcache_lock);
> }
> 
>-/**
>- * prune_dcache - shrink the dcache
>- * @count: number of entries to try and free
>- * @sb: if given, ignore dentries for other superblocks
>- *         which are being unmounted.
>- *
>- * Shrink the dcache. This is done when we need
>- * more memory, or simply when we need to unmount
>- * something (at which point we need to unuse
>- * all dentries).
>- *
>- * This function may fail to free any resources if
>- * all the dentries are in use.
>+/*
>+ * Shrink the dentry LRU on a given superblock.
>+ * @sb	 : superblock to shrink dentry LRU.
>+ * @count: If count is NULL, we prune all dentries on superblock.
>+ * @flags: If flags is noon-zero we need to do special processing based on
>+ * which flags are set. This means we don't need to maintain multiple
>+ * similar copies of this loop
>  */
>- 
>-static void prune_dcache(int count, struct super_block *sb)
>+static void __shrink_dcache_sb(struct super_block *sb, int *count, int flags)
> {
>+	LIST_HEAD(referenced);
>+	LIST_HEAD(tmp);
>+	struct dentry *dentry;
>+	int cnt = 0;
>+
>+	BUG_ON(!sb);
>+	BUG_ON((flags & DCACHE_REFERENCED) && count == NULL);
> 	spin_lock(&dcache_lock);
>-	for (; count ; count--) {
>-		struct dentry *dentry;
>-		struct list_head *tmp;
>-		struct rw_semaphore *s_umount;
>+	if (count != NULL)
>+		/* called from prune_dcache() and shrink_dcache_parent() */
>+		cnt = *count;
>+restart:
>+	if (count == NULL)
>+		list_splice_init(&sb->s_dentry_lru, &tmp);
>+	else {
>+		while (!list_empty(&sb->s_dentry_lru)) {
>+			dentry = list_entry(sb->s_dentry_lru.prev,
>+					    struct dentry, d_lru);
>+			BUG_ON(dentry->d_sb != sb);
> 
>-		cond_resched_lock(&dcache_lock);
>+			spin_lock(&dentry->d_lock);
> 
>-		tmp = dentry_unused.prev;
>-		if (sb) {
>-			/* Try to find a dentry for this sb, but don't try
>-			 * too hard, if they aren't near the tail they will
>-			 * be moved down again soon
>+			/*
>+			 * If we are honouring the DCACHE_REFERENCED flag and
>+			 * the dentry has this flag set, don't free it.  Clear
>+			 * the flag and put it back on the LRU
> 			 */
>-			int skip = count;
>-			while (skip && tmp != &dentry_unused &&
>-			    list_entry(tmp, struct dentry, d_lru)->d_sb != sb) {
>-				skip--;
>-				tmp = tmp->prev;
>+			if ((flags & DCACHE_REFERENCED)
>+			    && (dentry->d_flags & DCACHE_REFERENCED)) {
>+				dentry->d_flags &= ~DCACHE_REFERENCED;
>+				list_move_tail(&dentry->d_lru, &referenced);
>+				spin_unlock(&dentry->d_lock);
>+			} else {
>+				list_move_tail(&dentry->d_lru, &tmp);
>+				spin_unlock(&dentry->d_lock);
>+				cnt--;
>+				if (!cnt)
>+					break;
> 			}
> 		}
>-		if (tmp == &dentry_unused)
>-			break;
>-		list_del_init(tmp);
>-		prefetch(dentry_unused.prev);
>- 		dentry_stat.nr_unused--;
>-		dentry = list_entry(tmp, struct dentry, d_lru);
>-
>- 		spin_lock(&dentry->d_lock);
>+	}
>+	while (!list_empty(&tmp)) {
>+		dentry = list_entry(tmp.prev, struct dentry, d_lru);
>+		dentry_lru_del_init(dentry);
>+		spin_lock(&dentry->d_lock);
> 		/*
> 		 * We found an inuse dentry which was not removed from
>-		 * dentry_unused because of laziness during lookup.  Do not free
>-		 * it - just keep it off the dentry_unused list.
>+		 * the LRU because of laziness during lookup.  Do not free
>+		 * it - just keep it off the LRU list.
> 		 */
>- 		if (atomic_read(&dentry->d_count)) {
>- 			spin_unlock(&dentry->d_lock);
>+		if (atomic_read(&dentry->d_count)) {
>+			spin_unlock(&dentry->d_lock);
> 			continue;
> 		}
>-		/* If the dentry was recently referenced, don't free it. */
>-		if (dentry->d_flags & DCACHE_REFERENCED) {
>-			dentry->d_flags &= ~DCACHE_REFERENCED;
>- 			list_add(&dentry->d_lru, &dentry_unused);
>- 			dentry_stat.nr_unused++;
>- 			spin_unlock(&dentry->d_lock);
>+		prune_one_dentry(dentry);
>+		/* dentry->d_lock was dropped in prune_one_dentry() */
>+		cond_resched_lock(&dcache_lock);
>+	}
>+	if (count == NULL && !list_empty(&sb->s_dentry_lru))
>+		goto restart;
>+	if (count != NULL)
>+		*count = cnt;
>+	if (!list_empty(&referenced))
>+		list_splice(&referenced, &sb->s_dentry_lru);
>+	spin_unlock(&dcache_lock);
>+}
>+
>+/**
>+ * prune_dcache - shrink the dcache
>+ * @count: number of entries to try to free
>+ *
>+ * Shrink the dcache. This is done when we need more memory, or simply when we
>+ * need to unmount something (at which point we need to unuse all dentries).
>+ *
>+ * This function may fail to free any resources if all the dentries are in use.
>+ */
>+static void prune_dcache(int count)
>+{
>+	struct super_block *sb;
>+	int w_count;
>+	int unused = dentry_stat.nr_unused;
>+	int prune_ratio;
>+	int pruned;
>+
>+	if (unused == 0 || count == 0)
>+		return;
>+	spin_lock(&dcache_lock);
>+restart:
>+	if (count >= unused)
>+		prune_ratio = 1;
>+	else
>+		prune_ratio = unused / count;
>+	spin_lock(&sb_lock);
>+	list_for_each_entry(sb, &super_blocks, s_list) {
>+		if (sb->s_nr_dentry_unused == 0)
> 			continue;
>-		}
>-		/*
>-		 * If the dentry is not DCACHED_REFERENCED, it is time
>-		 * to remove it from the dcache, provided the super block is
>-		 * NULL (which means we are trying to reclaim memory)
>-		 * or this dentry belongs to the same super block that
>-		 * we want to shrink.
>+		sb->s_count++;
>+		/* We reclaim them same percentage from each superblock.
>+		 * We calculate number of dentries to scan on this sb
>+		 * as follows, but the implementation is arranged to avoid
>+		 * overflows:
>+		 * number of dentries to scan on this sb =
>+		 * count * (number of dentries on this sb /
>+		 * number of dentries in the machine)
> 		 */
>+		spin_unlock(&sb_lock);
>+		if (prune_ratio != 1)
>+			w_count = (sb->s_nr_dentry_unused / prune_ratio) + 1;
>+		else
>+			w_count = sb->s_nr_dentry_unused;
>+		pruned = w_count;
> 		/*
>-		 * If this dentry is for "my" filesystem, then I can prune it
>-		 * without taking the s_umount lock (I already hold it).
>+		 * We need to be sure this filesystem isn't being unmounted,
>+		 * otherwise we could race with generic_shutdown_super(), and
>+		 * end up holding a reference to an inode while the filesystem
>+		 * is unmounted.  So we try to get s_umount, and make sure
>+		 * s_root isn't NULL.
> 		 */
>-		if (sb && dentry->d_sb == sb) {
>-			prune_one_dentry(dentry);
>-			continue;
>+		if (down_read_trylock(&sb->s_umount)) {
>+			if ((sb->s_root != NULL) &&
>+			    (!list_empty(&sb->s_dentry_lru))) {
>+				spin_unlock(&dcache_lock);
>+				__shrink_dcache_sb(sb, &w_count,
>+						   DCACHE_REFERENCED);
>+				pruned -= w_count;
>+				spin_lock(&dcache_lock);
>+			}
>+			up_read(&sb->s_umount);
> 		}
>+		spin_lock(&sb_lock);
>+		count -= pruned;
> 		/*
>-		 * ...otherwise we need to be sure this filesystem isn't being
>-		 * unmounted, otherwise we could race with
>-		 * generic_shutdown_super(), and end up holding a reference to
>-		 * an inode while the filesystem is unmounted.
>-		 * So we try to get s_umount, and make sure s_root isn't NULL.
>-		 * (Take a local copy of s_umount to avoid a use-after-free of
>-		 * `dentry').
>+		 * restart only when sb is no longer on the list and
>+		 * we have more work to do.
> 		 */
>-		s_umount = &dentry->d_sb->s_umount;
>-		if (down_read_trylock(s_umount)) {
>-			if (dentry->d_sb->s_root != NULL) {
>-				prune_one_dentry(dentry);
>-				up_read(s_umount);
>-				continue;
>-			}
>-			up_read(s_umount);
>+		if (__put_super_and_need_restart(sb) && count > 0) {
>+			spin_unlock(&sb_lock);
>+			goto restart;
> 		}
>-		spin_unlock(&dentry->d_lock);
>-		/* Cannot remove the first dentry, and it isn't appropriate
>-		 * to move it to the head of the list, so give up, and try
>-		 * later
>-		 */
>-		break;
> 	}
>+	spin_unlock(&sb_lock);
> 	spin_unlock(&dcache_lock);
> }
> 
>-/*
>- * Shrink the dcache for the specified super block.
>- * This allows us to unmount a device without disturbing
>- * the dcache for the other devices.
>- *
>- * This implementation makes just two traversals of the
>- * unused list.  On the first pass we move the selected
>- * dentries to the most recent end, and on the second
>- * pass we free them.  The second pass must restart after
>- * each dput(), but since the target dentries are all at
>- * the end, it's really just a single traversal.
>- */
>-
> /**
>  * shrink_dcache_sb - shrink dcache for a superblock
>  * @sb: superblock
>@@ -510,44 +573,9 @@ static void prune_dcache(int count, stru
>  * is used to free the dcache before unmounting a file
>  * system
>  */
>-
> void shrink_dcache_sb(struct super_block * sb)
> {
>-	struct list_head *tmp, *next;
>-	struct dentry *dentry;
>-
>-	/*
>-	 * Pass one ... move the dentries for the specified
>-	 * superblock to the most recent end of the unused list.
>-	 */
>-	spin_lock(&dcache_lock);
>-	list_for_each_safe(tmp, next, &dentry_unused) {
>-		dentry = list_entry(tmp, struct dentry, d_lru);
>-		if (dentry->d_sb != sb)
>-			continue;
>-		list_move(tmp, &dentry_unused);
>-	}
>-
>-	/*
>-	 * Pass two ... free the dentries for this superblock.
>-	 */
>-repeat:
>-	list_for_each_safe(tmp, next, &dentry_unused) {
>-		dentry = list_entry(tmp, struct dentry, d_lru);
>-		if (dentry->d_sb != sb)
>-			continue;
>-		dentry_stat.nr_unused--;
>-		list_del_init(tmp);
>-		spin_lock(&dentry->d_lock);
>-		if (atomic_read(&dentry->d_count)) {
>-			spin_unlock(&dentry->d_lock);
>-			continue;
>-		}
>-		prune_one_dentry(dentry);
>-		cond_resched_lock(&dcache_lock);
>-		goto repeat;
>-	}
>-	spin_unlock(&dcache_lock);
>+	__shrink_dcache_sb(sb, NULL, 0);
> }
> 
> /*
>@@ -563,10 +591,7 @@ static void shrink_dcache_for_umount_sub
> 
> 	/* detach this root from the system */
> 	spin_lock(&dcache_lock);
>-	if (!list_empty(&dentry->d_lru)) {
>-		dentry_stat.nr_unused--;
>-		list_del_init(&dentry->d_lru);
>-	}
>+	dentry_lru_del_init(dentry);
> 	__d_drop(dentry);
> 	spin_unlock(&dcache_lock);
> 
>@@ -580,10 +605,7 @@ static void shrink_dcache_for_umount_sub
> 			spin_lock(&dcache_lock);
> 			list_for_each_entry(loop, &dentry->d_subdirs,
> 					    d_u.d_child) {
>-				if (!list_empty(&loop->d_lru)) {
>-					dentry_stat.nr_unused--;
>-					list_del_init(&loop->d_lru);
>-				}
>+				dentry_lru_del_init(loop);
> 
> 				__d_drop(loop);
> 				cond_resched_lock(&dcache_lock);
>@@ -761,17 +783,13 @@ resume:
> 		struct dentry *dentry = list_entry(tmp, struct dentry, d_u.d_child);
> 		next = tmp->next;
> 
>-		if (!list_empty(&dentry->d_lru)) {
>-			dentry_stat.nr_unused--;
>-			list_del_init(&dentry->d_lru);
>-		}
>+		dentry_lru_del_init(dentry);
> 		/* 
> 		 * move only zero ref count dentries to the end 
> 		 * of the unused list for prune_dcache
> 		 */
> 		if (!atomic_read(&dentry->d_count)) {
>-			list_add_tail(&dentry->d_lru, &dentry_unused);
>-			dentry_stat.nr_unused++;
>+			dentry_lru_add_tail(dentry);
> 			found++;
> 		}
> 
>@@ -813,10 +831,11 @@ out:
>  
> void shrink_dcache_parent(struct dentry * parent)
> {
>+	struct super_block *sb = parent->d_sb;
> 	int found;
> 
> 	while ((found = select_parent(parent)) != 0)
>-		prune_dcache(found, parent->d_sb);
>+		__shrink_dcache_sb(sb, &found, 0);
> }
> 
> /*
>@@ -836,7 +855,7 @@ static int shrink_dcache_memory(int nr, 
> 	if (nr) {
> 		if (!(gfp_mask & __GFP_FS))
> 			return -1;
>-		prune_dcache(nr, NULL);
>+		prune_dcache(nr);
> 	}
> 	return (dentry_stat.nr_unused / 100) * sysctl_vfs_cache_pressure;
> }
>@@ -1184,7 +1203,7 @@ struct dentry *d_splice_alias(struct ino
>  * rcu_read_lock() and rcu_read_unlock() are used to disable preemption while
>  * lookup is going on.
>  *
>- * dentry_unused list is not updated even if lookup finds the required dentry
>+ * The dentry unused LRU is not updated even if lookup finds the required dentry
>  * in there. It is updated in places such as prune_dcache, shrink_dcache_sb,
>  * select_parent and __dget_locked. This laziness saves lookup from dcache_lock
>  * acquisition.
>Index: kernel/fs/super.c
>===================================================================
>--- kernel.orig/fs/super.c
>+++ kernel/fs/super.c
>@@ -71,6 +71,7 @@ static struct super_block *alloc_super(s
> 		INIT_LIST_HEAD(&s->s_instances);
> 		INIT_HLIST_HEAD(&s->s_anon);
> 		INIT_LIST_HEAD(&s->s_inodes);
>+		INIT_LIST_HEAD(&s->s_dentry_lru);
> 		init_rwsem(&s->s_umount);
> 		mutex_init(&s->s_lock);
> 		lockdep_set_class(&s->s_umount, &type->s_umount_key);
>Index: kernel/include/linux/fs.h
>===================================================================
>--- kernel.orig/include/linux/fs.h
>+++ kernel/include/linux/fs.h
>@@ -946,6 +946,11 @@ struct super_block {
> 	/* Granularity of c/m/atime in ns.
> 	   Cannot be worse than a second */
> 	u32		   s_time_gran;
>+#ifndef __GENKSYMS__
>+	/* s_dentry_lru and s_nr_dentry_unused are protected by dcache_lock */
>+	struct list_head	s_dentry_lru;	/* unused dentry lru */
>+	int			s_nr_dentry_unused;	/* # of dentry on lru */
>+#endif
> };
> 
> extern struct timespec current_fs_time(struct super_block *sb);

Actions: View | Diff

Attachments on bug 450194: 308995 | 915143