12:27 a.m.
Greetings,
0day kernel testing robot got the below dmesg and the first bad commit is
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git master
commit 0e1cc95b4cc7293bb7b39175035e7f7e45c90977
Author: Mel Gorman <mgorman(a)suse.de>
AuthorDate: Tue Jun 30 14:57:27 2015 -0700
Commit: Linus Torvalds <torvalds(a)linux-foundation.org>
CommitDate: Tue Jun 30 19:44:56 2015 -0700
mm: meminit: finish initialisation of struct pages before basic setup
Waiman Long reported that 24TB machines hit OOM during basic setup when
struct page initialisation was deferred. One approach is to initialise
memory on demand but it interferes with page allocator paths. This patch
creates dedicated threads to initialise memory before basic setup. It
then blocks on a rw_semaphore until completion as a wait_queue and counter
is overkill. This may be slower to boot but it's simplier overall and
also gets rid of a section mangling which existed so kswapd could do the
initialisation.
[akpm(a)linux-foundation.org: include rwsem.h, use DECLARE_RWSEM, fix comment, remove
unneeded cast]
Signed-off-by: Mel Gorman <mgorman(a)suse.de>
Cc: Waiman Long <waiman.long(a)hp.com
Cc: Nathan Zimmer <nzimmer(a)sgi.com>
Cc: Dave Hansen <dave.hansen(a)intel.com>
Cc: Scott Norton <scott.norton(a)hp.com>
Tested-by: Daniel J Blueman <daniel(a)numascale.com>
Signed-off-by: Andrew Morton <akpm(a)linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds(a)linux-foundation.org>
+-----------------------------------------------------+------------+------------+-----------------+
| | 74033a798f | 0e1cc95b4c |
v4.2-rc1_070709 |
+-----------------------------------------------------+------------+------------+-----------------+
| boot_successes | 63 | 0 | 0
|
| boot_failures | 0 | 24 | 13
|
| WARNING:at_kernel/locking/lockdep.c:#lock_release() | 0 | 24 | 13
|
| backtrace:up_read | 0 | 24 | 13
|
| backtrace:deferred_init_memmap | 0 | 24 | 13
|
+-----------------------------------------------------+------------+------------+-----------------+
[ 0.240862] kvm-stealtime: cpu 1, msr 1100d140
[ 0.260270] ------------[ cut here ]------------
[ 0.260270] ------------[ cut here ]------------
[ 0.261323] WARNING: CPU: 1 PID: 15 at kernel/locking/lockdep.c:3382
lock_release+0xde/0x3d3()
[ 0.261323] WARNING: CPU: 1 PID: 15 at kernel/locking/lockdep.c:3382
lock_release+0xde/0x3d3()
[ 0.264199] DEBUG_LOCKS_WARN_ON(depth <= 0)
[ 0.264199] DEBUG_LOCKS_WARN_ON(depth <= 0)
[ 0.265313] CPU: 1 PID: 15 Comm: pgdatinit0 Not tainted 4.1.0-11369-g0e1cc95 #1
[ 0.265313] CPU: 1 PID: 15 Comm: pgdatinit0 Not tainted 4.1.0-11369-g0e1cc95 #1
[ 0.269739] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS
1.7.5-20140531_083030-gandalf 04/01/2014
[ 0.269739] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS
1.7.5-20140531_083030-gandalf 04/01/2014
[ 0.270000] 0000000000000009
[ 0.270000] 0000000000000009 ffff88000f997c48 ffff88000f997c48 ffffffff817d5686
ffffffff817d5686 ffffffff810cb07d ffffffff810cb07d
[ 0.270000] ffff88000f997c98
[ 0.270000] ffff88000f997c98 ffff88000f997c88 ffff88000f997c88 ffffffff8108bd43
ffffffff8108bd43 ffff88000f997c68 ffff88000f997c68
[ 0.270000] ffffffff810c3b8d
[ 0.270000] ffffffff810c3b8d ffff88000f990000 ffff88000f990000 ffff8800114fe000
ffff8800114fe000 ffffffff8236e530 ffffffff8236e530
[ 0.270000] Call Trace:
[ 0.270000] Call Trace:
[ 0.270000] [<ffffffff817d5686>] dump_stack+0x4c/0x6e
[ 0.270000] [<ffffffff817d5686>] dump_stack+0x4c/0x6e
[ 0.270000] [<ffffffff810cb07d>] ? console_unlock+0x3f3/0x422
[ 0.270000] [<ffffffff810cb07d>] ? console_unlock+0x3f3/0x422
[ 0.270000] [<ffffffff8108bd43>] warn_slowpath_common+0x9c/0xb6
[ 0.270000] [<ffffffff8108bd43>] warn_slowpath_common+0x9c/0xb6
[ 0.270000] [<ffffffff810c3b8d>] ? lock_release+0xde/0x3d3
[ 0.270000] [<ffffffff810c3b8d>] ? lock_release+0xde/0x3d3
[ 0.270000] [<ffffffff821dc011>] ? deferred_init_memmap+0x306/0x32d
[ 0.270000] [<ffffffff821dc011>] ? deferred_init_memmap+0x306/0x32d
[ 0.270000] [<ffffffff8108bdc2>] warn_slowpath_fmt+0x46/0x48
[ 0.270000] [<ffffffff8108bdc2>] warn_slowpath_fmt+0x46/0x48
[ 0.270000] [<ffffffff810c3b8d>] lock_release+0xde/0x3d3
[ 0.270000] [<ffffffff810c3b8d>] lock_release+0xde/0x3d3
[ 0.270000] [<ffffffff821dbd0b>] ? deferred_free_range+0x63/0x63
[ 0.270000] [<ffffffff821dbd0b>] ? deferred_free_range+0x63/0x63
[ 0.270000] [<ffffffff810c08ac>] up_read+0x20/0x2c
[ 0.270000] [<ffffffff810c08ac>] up_read+0x20/0x2c
[ 0.270000] [<ffffffff821dc011>] deferred_init_memmap+0x306/0x32d
[ 0.270000] [<ffffffff821dc011>] deferred_init_memmap+0x306/0x32d
[ 0.270000] [<ffffffff817d9ea9>] ? __schedule+0x3b9/0x5af
[ 0.270000] [<ffffffff817d9ea9>] ? __schedule+0x3b9/0x5af
[ 0.270000] [<ffffffff821dbd0b>] ? deferred_free_range+0x63/0x63
[ 0.270000] [<ffffffff821dbd0b>] ? deferred_free_range+0x63/0x63
[ 0.270000] [<ffffffff810a2cb8>] kthread+0xe0/0xe8
[ 0.270000] [<ffffffff810a2cb8>] kthread+0xe0/0xe8
[ 0.270000] [<ffffffff817de2eb>] ? _raw_spin_unlock_irq+0x32/0x46
[ 0.270000] [<ffffffff817de2eb>] ? _raw_spin_unlock_irq+0x32/0x46
[ 0.270000] [<ffffffff810a2bd8>] ? __kthread_parkme+0xad/0xad
[ 0.270000] [<ffffffff810a2bd8>] ? __kthread_parkme+0xad/0xad
[ 0.270000] [<ffffffff817df1cf>] ret_from_fork+0x3f/0x70
[ 0.270000] [<ffffffff817df1cf>] ret_from_fork+0x3f/0x70
[ 0.270000] [<ffffffff810a2bd8>] ? __kthread_parkme+0xad/0xad
[ 0.270000] [<ffffffff810a2bd8>] ? __kthread_parkme+0xad/0xad
[ 0.270000] ---[ end trace 95f3f95a9e2dd516 ]---
[ 0.270000] ---[ end trace 95f3f95a9e2dd516 ]---
git bisect start 9bbbdce9ba479ee60d157b45e83c2060f5e6ceac v4.1 --
git bisect good f822dcc63f966fc79b11a8254fa0942b1aa8c71e # 14:45 20+ 0 Merge
tag 'sound-fix-4.2-rc1' of
git://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound
git bisect bad daa259f637dc2e674b38884342f0ded0189aceff # 15:40 0- 11 Merge
'mediatek/v4.2-next/soc' into devel-hourly-2015070709
git bisect bad 13d45f79a2af84de9083310db58b309a61065208 # 16:14 0- 3 Merge
branch 'for-next' of
git://git.kernel.org/pub/scm/linux/kernel/git/cooloney/linux-leds
git bisect good d5fb82137b6cd39e67c4321f4f5ce9b03d4d04e6 # 16:23 22+ 0 Merge
branch 'irq-urgent-for-linus' of
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
git bisect bad 2d01eedf1d14432f4db5388a49dc5596a8c5bd02 # 16:29 0- 4 Merge
branch 'akpm' (patches from Andrew)
git bisect good 6ac15baacb6ecd87c66209627753b96ded3b4515 # 17:36 21+ 0 Merge
branch 'timers-urgent-for-linus' of
git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
git bisect bad 9ce71148b027e2bd27016139cae1c39401587695 # 19:04 0- 13 devpts:
if initialization failed, don't crash when opening /dev/ptmx
git bisect bad 460b865e53c347ebf110e50d499718cd9b39d810 # 20:05 0- 2 fs:
document seq_open()'s usage of file->private_data
git bisect good 7e18adb4f80bea90d30b62158694d97c31f71d37 # 20:14 20+ 0 mm:
meminit: initialise remaining struct pages in parallel with kswapd
git bisect good ac5d2539b2382689b1cdb90bd60dcd49f61c2773 # 21:18 22+ 0 mm:
meminit: reduce number of times pageblocks are set during struct page init
git bisect bad 0e1cc95b4cc7293bb7b39175035e7f7e45c90977 # 21:26 0- 12 mm:
meminit: finish initialisation of struct pages before basic setup
git bisect good 74033a798f5a5db368126ee6f690111cf019bf7a # 21:37 22+ 0 mm:
meminit: remove mminit_verify_page_links
# first bad commit: [0e1cc95b4cc7293bb7b39175035e7f7e45c90977] mm: meminit: finish
initialisation of struct pages before basic setup
git bisect good 74033a798f5a5db368126ee6f690111cf019bf7a # 21:41 63+ 0 mm:
meminit: remove mminit_verify_page_links
# extra tests with DEBUG_INFO
git bisect bad 0e1cc95b4cc7293bb7b39175035e7f7e45c90977 # 21:50 0- 5 mm:
meminit: finish initialisation of struct pages before basic setup
# extra tests on HEAD of linux-devel/devel-hourly-2015070709
git bisect bad 9bbbdce9ba479ee60d157b45e83c2060f5e6ceac # 21:50 0- 13 0day
head guard for 'devel-hourly-2015070709'
# extra tests on tree/branch linus/master
# extra tests with first bad commit reverted
# extra tests on tree/branch linus/master
# extra tests on tree/branch next/master
git bisect bad 8e551e96341aa57779ea93a87c857ae61a057f11 # 22:27 0- 1 Add
linux-next specific files for 20150707
This script may reproduce the error.
----------------------------------------------------------------------------
#!/bin/bash
kernel=$1
initrd=quantal-core-x86_64.cgz
wget --no-clobber
https://github.com/fengguang/reproduce-kernel-bug/raw/master/initrd/$initrd
kvm=(
qemu-system-x86_64
-enable-kvm
-cpu kvm64
-kernel $kernel
-initrd $initrd
-m 300
-smp 2
-device e1000,netdev=net0
-netdev user,id=net0
-boot order=nc
-no-reboot
-watchdog i6300esb
-rtc base=localtime
-serial stdio
-display none
-monitor null
)
append=(
hung_task_panic=1
earlyprintk=ttyS0,115200
systemd.log_level=err
debug
apic=debug
sysrq_always_enabled
rcupdate.rcu_cpu_stall_timeout=100
panic=-1
softlockup_panic=1
nmi_watchdog=panic
oops=panic
load_ramdisk=2
prompt_ramdisk=0
console=ttyS0,115200
console=tty0
vga=normal
root=/dev/ram0
rw
drbd.minor_count=8
)
"${kvm[@]}" --append "${append[*]}"
----------------------------------------------------------------------------
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/lkp Intel Corporation
8:32 p.m.
On Tue, Jul 07, 2015 at 10:27:58PM +0800, kernel test robot wrote:
Greetings,
0day kernel testing robot got the below dmesg and the first bad commit is
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git master
commit 0e1cc95b4cc7293bb7b39175035e7f7e45c90977
Author: Mel Gorman <mgorman(a)suse.de>
AuthorDate: Tue Jun 30 14:57:27 2015 -0700
Commit: Linus Torvalds <torvalds(a)linux-foundation.org>
CommitDate: Tue Jun 30 19:44:56 2015 -0700
mm: meminit: finish initialisation of struct pages before basic setup
Waiman Long reported that 24TB machines hit OOM during basic setup when
struct page initialisation was deferred. One approach is to initialise
memory on demand but it interferes with page allocator paths. This patch
creates dedicated threads to initialise memory before basic setup. It
then blocks on a rw_semaphore until completion as a wait_queue and counter
is overkill. This may be slower to boot but it's simplier overall and
also gets rid of a section mangling which existed so kswapd could do the
initialisation.
[akpm(a)linux-foundation.org: include rwsem.h, use DECLARE_RWSEM, fix comment, remove
unneeded cast]
Signed-off-by: Mel Gorman <mgorman(a)suse.de>
Cc: Waiman Long <waiman.long(a)hp.com
Cc: Nathan Zimmer <nzimmer(a)sgi.com>
Cc: Dave Hansen <dave.hansen(a)intel.com>
Cc: Scott Norton <scott.norton(a)hp.com>
Tested-by: Daniel J Blueman <daniel(a)numascale.com>
Signed-off-by: Andrew Morton <akpm(a)linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds(a)linux-foundation.org>
Would it be possible to test with this patch on top?
---8<---
From: Nicolai Stange <nicstange(a)gmail.com>
Subject: Re: [PATCH] mm/page_alloc: deferred meminit: replace rwsem with completion
Commit 0e1cc95b4cc7
("mm: meminit: finish initialisation of struct pages before basic setup")
introduced a rwsem to signal completion of the initialization workers.
Lockdep complains about possible recursive locking:
=============================================
[ INFO: possible recursive locking detected ]
4.1.0-12802-g1dc51b8 #3 Not tainted
---------------------------------------------
swapper/0/1 is trying to acquire lock:
(pgdat_init_rwsem){++++.+},
at: [<ffffffff8424c7fb>] page_alloc_init_late+0xc7/0xe6
but task is already holding lock:
(pgdat_init_rwsem){++++.+},
at: [<ffffffff8424c772>] page_alloc_init_late+0x3e/0xe6
Replace the rwsem by a completion together with an atomic
"outstanding work counter".
Signed-off-by: Nicolai Stange <nicstange(a)gmail.com>
Acked-by: Mel Gorman <mgorman(a)suse.de>
---
mm/page_alloc.c | 34 +++++++++++++++++++++++++++-------
1 file changed, 27 insertions(+), 7 deletions(-)
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 506eac8..3886e66 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -18,7 +18,9 @@
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/interrupt.h>
-#include <linux/rwsem.h>
+#include <linux/completion.h>
+#include <linux/atomic.h>
+#include <asm/barrier.h>
#include <linux/pagemap.h>
#include <linux/jiffies.h>
#include <linux/bootmem.h>
@@ -1062,7 +1064,20 @@ static void __init deferred_free_range(struct page *page,
__free_pages_boot_core(page, pfn, 0);
}
-static __initdata DECLARE_RWSEM(pgdat_init_rwsem);
+/* counter and completion tracking outstanding deferred_init_memmap()
+ threads */
+static atomic_t pgdat_init_n_undone __initdata;
+static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
+
+static inline void __init pgdat_init_report_one_done(void)
+{
+ /* Write barrier is paired with read barrier in
+ page_alloc_init_late(). It makes all writes visible to
+ readers seeing our decrement on pgdat_init_n_undone. */
+ smp_wmb();
+ if (atomic_dec_and_test(&pgdat_init_n_undone))
+ complete(&pgdat_init_all_done_comp);
+}
/* Initialise remaining memory on a node */
static int __init deferred_init_memmap(void *data)
@@ -1079,7 +1094,7 @@ static int __init deferred_init_memmap(void *data)
const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
if (first_init_pfn == ULONG_MAX) {
- up_read(&pgdat_init_rwsem);
+ pgdat_init_report_one_done();
return 0;
}
@@ -1179,7 +1194,8 @@ free_range:
pr_info("node %d initialised, %lu pages in %ums\n", nid, nr_pages,
jiffies_to_msecs(jiffies - start));
- up_read(&pgdat_init_rwsem);
+
+ pgdat_init_report_one_done();
return 0;
}
@@ -1187,14 +1203,18 @@ void __init page_alloc_init_late(void)
{
int nid;
+ /* There will be num_node_state(N_MEMORY) threads */
+ atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
for_each_node_state(nid, N_MEMORY) {
- down_read(&pgdat_init_rwsem);
kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
}
/* Block until all are initialised */
- down_write(&pgdat_init_rwsem);
- up_write(&pgdat_init_rwsem);
+ wait_for_completion(&pgdat_init_all_done_comp);
+
+ /* Paired with write barrier in deferred_init_memmap(),
+ ensures a consistent view of all its writes. */
+ smp_rmb();
}
#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
--
2.4.5
1:25 a.m.
On Wed, Jul 08, 2015 at 11:32:13AM +0100, Mel Gorman wrote:
From: Nicolai Stange <nicstange(a)gmail.com>
Subject: Re: [PATCH] mm/page_alloc: deferred meminit: replace rwsem with completion
Commit 0e1cc95b4cc7
("mm: meminit: finish initialisation of struct pages before basic setup")
introduced a rwsem to signal completion of the initialization workers.
Lockdep complains about possible recursive locking:
=============================================
[ INFO: possible recursive locking detected ]
4.1.0-12802-g1dc51b8 #3 Not tainted
---------------------------------------------
swapper/0/1 is trying to acquire lock:
(pgdat_init_rwsem){++++.+},
at: [<ffffffff8424c7fb>] page_alloc_init_late+0xc7/0xe6
but task is already holding lock:
(pgdat_init_rwsem){++++.+},
at: [<ffffffff8424c772>] page_alloc_init_late+0x3e/0xe6
Replace the rwsem by a completion together with an atomic
"outstanding work counter".
Signed-off-by: Nicolai Stange <nicstange(a)gmail.com>
Acked-by: Mel Gorman <mgorman(a)suse.de>
---
mm/page_alloc.c | 34 +++++++++++++++++++++++++++-------
1 file changed, 27 insertions(+), 7 deletions(-)
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 506eac8..3886e66 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -18,7 +18,9 @@
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/interrupt.h>
-#include <linux/rwsem.h>
+#include <linux/completion.h>
+#include <linux/atomic.h>
+#include <asm/barrier.h>
#include <linux/pagemap.h>
#include <linux/jiffies.h>
#include <linux/bootmem.h>
@@ -1062,7 +1064,20 @@ static void __init deferred_free_range(struct page *page,
__free_pages_boot_core(page, pfn, 0);
}
-static __initdata DECLARE_RWSEM(pgdat_init_rwsem);
+/* counter and completion tracking outstanding deferred_init_memmap()
+ threads */
Wrong comment style.
+static atomic_t pgdat_init_n_undone __initdata;
+static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
+
+static inline void __init pgdat_init_report_one_done(void)
+{
+ /* Write barrier is paired with read barrier in
+ page_alloc_init_late(). It makes all writes visible to
+ readers seeing our decrement on pgdat_init_n_undone. */
Wrong comment style.
+ smp_wmb();
Pointless barrier, because
+ if (atomic_dec_and_test(&pgdat_init_n_undone))
implies a full memory barrier, furthermore see below.
+ complete(&pgdat_init_all_done_comp);
+}
/* Initialise remaining memory on a node */
static int __init deferred_init_memmap(void *data)
@@ -1187,14 +1203,18 @@ void __init page_alloc_init_late(void)
{
int nid;
+ /* There will be num_node_state(N_MEMORY) threads */
+ atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
for_each_node_state(nid, N_MEMORY) {
- down_read(&pgdat_init_rwsem);
kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
}
/* Block until all are initialised */
- down_write(&pgdat_init_rwsem);
- up_write(&pgdat_init_rwsem);
+ wait_for_completion(&pgdat_init_all_done_comp);
+
+ /* Paired with write barrier in deferred_init_memmap(),
+ ensures a consistent view of all its writes. */
Wrong comment style
+ smp_rmb();
Wrong barrier, IF you want a barrier it should be before
wait_for_completion, such that if you observe complete, you then must
also observe whatever happened prior to the completion.
But I would argue a completion had better imply that anyway.
}
#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
2629
days inactive
2630
days old
2 comments
3 participants
participants (3)
-
kernel test robot -
Mel Gorman -
Peter Zijlstra