KKDDpanic: ufs_dirbad: /a: bad dir ino 3370734 at ofBBfset 0: mangled entry cpuid = 1 KDB: enter: panic [thread pid 768 tid 100104 ] Stopped at kdb_enter_why+0x3a: movl $0,kdb_why db> trace Tracing pid 768 tid 100104 td 0xc74c6690 kdb_enter_why(c0b3b0a1,c0b3b0a1,c0b5d0ec,ed2dd85c,1,...) at kdb_enter_why+0x3a panic(c0b5d0ec,c70365c0,336eee,0,c0b5d164,...) at panic+0x136 ufs_dirbad(c9c40180,0,c0b5d164,0,ed2dd904,...) at ufs_dirbad+0x49 ufs_lookup(ed2dd94c,c0b6f08d,cb090160,cb090160,ed2ddba4,...) at ufs_lookup+0x377 VOP_CACHEDLOOKUP_APV(c0c0c1a0,ed2dd94c,ed2ddba4,ed2ddb90,c75dae00,...) at VOP_CACHEDLOOKUP_APV+0xc5 vfs_cache_lookup(ed2dd9d0,c0b471e4,cb090160,2,cb090160,...) at vfs_cache_lookup+0xd3 VOP_LOOKUP_APV(c0c0c1a0,ed2dd9d0,c74c6690,c0b46172,19b,...) at VOP_LOOKUP_APV+0xe5 lookup(ed2ddb7c,c0b46172,c6,bf,c703a72c,...) at lookup+0x59e namei(ed2ddb7c,c74c6690,ed2dda74,246,c0c61ab8,...) at namei+0x35e vn_open_cred(ed2ddb7c,ed2ddc78,1a4,c75dae00,c74871c8,...) at vn_open_cred+0x62 vn_open(ed2ddb7c,ed2ddc78,1a4,c74871c8,0,...) at vn_open+0x33 kern_open(c74c6690,8164100,0,a02,1a4,...) at kern_open+0xe7 open(c74c6690,ed2ddcfc,c,c0b40d50,c0beb278,...) at open+0x30 syscall(ed2ddd38) at syscall+0x2d3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (5, FreeBSD ELF32, open), eip = 0x281a161b, esp = 0xbfbfe8cc, ebp = 0xbfbfe8f8 --- db> ps pid ppid pgrp uid state wmesg wchan cmd 768 728 768 0 R+ CPU 1 bsdtar 766 728 766 0 R+ CPU 4 bsdtar 728 724 728 0 Ss+ ttyin 0xc68ce810 csh 724 645 724 0 Ss select 0xc0ca821c sshd 720 711 720 0 S+ ttyin 0xc69a7810 csh 719 1 719 0 Ss+ ttyin 0xc69db410 getty 718 1 718 0 Ss+ ttyin 0xc69dc810 getty 717 1 717 0 Ss+ ttyin 0xc69de410 getty 716 1 716 0 Ss+ ttyin 0xc69d3810 getty 715 1 715 0 Ss+ ttyin 0xc69d5810 getty 714 1 714 0 Ss+ ttyin 0xc69d5c10 getty 713 1 713 0 Ss+ ttyin 0xc69dc010 getty 712 1 712 0 Ss+ ttyin 0xc69db010 getty 711 1 711 0 Ss+ wait 0xc6feb2b8 login 672 1 672 0 Ss nanslp 0xc0c56844 cron 666 1 666 25 Ss pause 0xc7049b40 sendmail 660 1 660 0 Ss select 0xc0ca821c sendmail 645 1 645 0 Ss select 0xc0ca821c sshd 527 1 527 0 Ss select 0xc0ca821c syslogd 473 1 473 0 Ss select 0xc0ca821c devd 143 1 143 0 Ss pause 0xc704ab40 adjkerntz 56 0 0 0 SL sdflush 0xc0cb4668 [softdepflush] 55 0 0 0 SL syncer 0xc0c5666c [syncer] 54 0 0 0 SL vlruwt 0xc6fec000 [vnlru] 53 0 0 0 RL CPU 5 [bufdaemon] 52 0 0 0 SL pgzero 0xc0cb5220 [pagezero] 51 0 0 0 SL psleep 0xc0cb4e38 [vmdaemon] 50 0 0 0 SL psleep 0xc0cb4e00 [pagedaemon] 49 0 0 0 SL waiting_ 0xc0caa434 [sctp_iterator] 48 0 0 0 WL [irq1: atkbd0] 47 0 0 0 WL [swi0: sio] 46 0 0 0 WL [irq15: ata1] 45 0 0 0 WL [irq14: ata0] 44 0 0 0 SL usbevt 0xc68a8210 [usb4] 43 0 0 0 SL usbevt 0xc6994210 [usb3] 42 0 0 0 SL usbevt 0xc6977210 [usb2] 41 0 0 0 SL usbevt 0xc697c210 [usb1] 40 0 0 0 WL [irq20: uhci1 uhci3] 39 0 0 0 SL usbtsk 0xc0c53e14 [usbtask-dr] 38 0 0 0 SL usbtsk 0xc0c53e00 [usbtask-hc] 37 0 0 0 SL usbevt 0xc692d210 [usb0] 36 0 0 0 WL [irq21: uhci0 uhci+] 35 0 0 0 WL [irq259: bce1] 34 0 0 0 SL ispf 0xc67e14d4 [isp1: fc_thrd] 33 0 0 0 WL [irq258: isp1] 32 0 0 0 SL ispf 0xc67e06d4 [isp0: fc_thrd] 31 0 0 0 RL [irq257: isp0] 30 0 0 0 WL [irq16: mfi0] 29 0 0 0 WL [irq256: bce0] 28 0 0 0 WL [irq9: acpi0] 27 0 0 0 SL - 0xc685e900 [thread taskq] 26 0 0 0 WL [swi5: +] 9 0 0 0 SL - 0xc685eb00 [kqueue taskq] 8 0 0 0 SL - 0xc685eb80 [acpi_task_2] 7 0 0 0 SL - 0xc685eb80 [acpi_task_1] 6 0 0 0 SL - 0xc685eb80 [acpi_task_0] 25 0 0 0 WL [swi2: cambio] 5 0 0 0 SL ccb_scan 0xc0c24394 [xpt_thrd] 24 0 0 0 WL [swi6: task queue] 23 0 0 0 WL [swi6: Giant taskq] 22 0 0 0 SL - 0xc0c56674 [yarrow] 4 0 0 0 SL - 0xc0c5420c [g_down] 3 0 0 0 RL CPU 3 [g_up] 2 0 0 0 SL - 0xc0c54200 [g_event] 21 0 0 0 WL [swi3: vm] 20 0 0 0 WL [swi4: clock sio] 19 0 0 0 WL [swi1: net] 18 0 0 0 RL CPU 0 [idle: cpu0] 17 0 0 0 RL [idle: cpu1] 16 0 0 0 RL CPU 2 [idle: cpu2] 15 0 0 0 RL [idle: cpu3] 14 0 0 0 RL [idle: cpu4] 13 0 0 0 RL [idle: cpu5] 12 0 0 0 RL CPU 6 [idle: cpu6] 11 0 0 0 RL CPU 7 [idle: cpu7] 1 0 1 0 SLs wait 0xc6793ae0 [init] 10 0 0 0 SL audit_wo 0xc0cb40a4 [audit] 0 0 0 0 SLs sched 0xc0c542c0 [swapper] db> show pcpu cpuid = 1 curthread = 0xc74c6690: pid 768 "bsdtar" curpcb = 0xed2ddd90 fpcurthread = none idlethread = 0xc67a7460: pid 17 "idle: cpu1" APIC ID = 1 currentldt = 0x50 spin locks held: db> show allcpu No such command db> show allcpus No such command db> show all registers breaks threads proc thread watches pciregs geom cpusets files file intrcnt intr malloc pgrpdump conifhk lock allpcpu pcpu msgbuf allrman rman sleepqueue sleepq locktree sleepchain allchains lockchain turnstile witness alllocks locks cbstat domain protosw sockbuf socket unpcb lockedbufs buffer vnode lockedvnods inpcb tcpcb uma procvm map vmopag object vmochk pageq page freepages cyrixreg irqs lapic apic sysregs idt rtc db> show allpcpu Current CPU: 1 cpuid = 0 curthread = 0xc67a7230: pid 18 "idle: cpu0" curpcb = 0xe5424d90 fpcurthread = none idlethread = 0xc67a7230: pid 18 "idle: cpu0" APIC ID = 0 currentldt = 0x50 spin locks held: cpuid = 1 curthread = 0xc74c6690: pid 768 "bsdtar" curpcb = 0xed2ddd90 fpcurthread = none idlethread = 0xc67a7460: pid 17 "idle: cpu1" APIC ID = 1 currentldt = 0x50 spin locks held: cpuid = 2 curthread = 0xc6795000: pid 16 "idle: cpu2" curpcb = 0xe541ed90 fpcurthread = none idlethread = 0xc6795000: pid 16 "idle: cpu2" APIC ID = 2 currentldt = 0x50 spin locks held: cpuid = 3 curthread = 0xc6797690: pid 3 "g_up" curpcb = 0xe6e36d90 fpcurthread = none idlethread = 0xc6795230: pid 15 "idle: cpu3" APIC ID = 3 currentldt = 0x50 spin locks held: cpuid = 4 curthread = 0xc75c3690: pid 766 "bsdtar" curpcb = 0xed348d90 fpcurthread = none idlethread = 0xc6795460: pid 14 "idle: cpu4" APIC ID = 4 currentldt = 0x50 spin locks held: cpuid = 5 curthread = 0xc6fea8c0: pid 53 "bufdaemon" curpcb = 0xeafead90 fpcurthread = none idlethread = 0xc6795690: pid 13 "idle: cpu5" APIC ID = 5 currentldt = 0x50 spin locks held: cpuid = 6 curthread = 0xc67958c0: pid 12 "idle: cpu6" curpcb = 0xe5412d90 fpcurthread = none idlethread = 0xc67958c0: pid 12 "idle: cpu6" APIC ID = 6 currentldt = 0x50 spin locks held: cpuid = 7 curthread = 0xc6795af0: pid 11 "idle: cpu7" curpcb = 0xe540fd90 fpcurthread = none idlethread = 0xc6795af0: pid 11 "idle: cpu7" APIC ID = 7 currentldt = 0x50 spin locks held: db> show locks db> show alllocks Process 4 (g_down) thread 0xc6797460 (100015) exclusive sleep mutex isp r = 0 (0xc67e0690) locked @ /usr/src/sys/cam/cam_periph.h:182 db> show lockedvnods Locked vnodes 0xcb090160: tag ufs, type VDIR usecount 1, writecount 0, refcount 3 mountedhere 0 flags () v_object 0xcaa70c1c ref 0 pages 1 lock type ufs: EXCL (count 1) by thread 0xc74c6690 (pid 768)#0 0xc077f6c0 at _lockmgr+0x560 #1 0xc099cb51 at ffs_lock+0x91 #2 0xc0aafcc5 at VOP_LOCK1_APV+0xa5 #3 0xc081f7b8 at _vn_lock+0xf8 #4 0xc0813c20 at vget+0x110 #5 0xc08029b7 at cache_lookup+0x4e7 #6 0xc0802aea at vfs_cache_lookup+0xaa #7 0xc0ab0e65 at VOP_LOOKUP_APV+0xe5 #8 0xc0808d2e at lookup+0x59e #9 0xc08098de at namei+0x35e #10 0xc081f022 at vn_open_cred+0x62 #11 0xc081f513 at vn_open+0x33 #12 0xc081e177 at kern_open+0xe7 #13 0xc081e660 at open+0x30 #14 0xc0a98853 at syscall+0x2d3 #15 0xc0a7ea80 at Xint0x80_syscall+0x20 ino 3370734, on dev da0s1d 0xcb5906e0: tag ufs, type VDIR usecount 0, writecount 0, refcount 2 mountedhere 0 flags () v_object 0xc9b65a2c ref 0 pages 0 lock type ufs: EXCL (count 1) by thread 0xc6fea8c0 (pid 53)#0 0xc077f6c0 at _lockmgr+0x560 #1 0xc099cb51 at ffs_lock+0x91 #2 0xc0aafcc5 at VOP_LOCK1_APV+0xa5 #3 0xc081f7b8 at _vn_lock+0xf8 #4 0xc07ff5d8 at flushbufqueues+0x568 #5 0xc07ff8f0 at buf_daemon+0xc0 #6 0xc0770ea8 at fork_exit+0xb8 #7 0xc0a7ea90 at fork_trampoline+0x8 ino 24168915, on dev da0s1d db> trace 768 Tracing pid 768 tid 100104 td 0xc74c6690 kdb_enter_why(c0b3b0a1,c0b3b0a1,c0b5d0ec,ed2dd85c,1,...) at kdb_enter_why+0x3a panic(c0b5d0ec,c70365c0,336eee,0,c0b5d164,...) at panic+0x136 ufs_dirbad(c9c40180,0,c0b5d164,0,ed2dd904,...) at ufs_dirbad+0x49 ufs_lookup(ed2dd94c,c0b6f08d,cb090160,cb090160,ed2ddba4,...) at ufs_lookup+0x377 VOP_CACHEDLOOKUP_APV(c0c0c1a0,ed2dd94c,ed2ddba4,ed2ddb90,c75dae00,...) at VOP_CACHEDLOOKUP_APV+0xc5 vfs_cache_lookup(ed2dd9d0,c0b471e4,cb090160,2,cb090160,...) at vfs_cache_lookup+0xd3 VOP_LOOKUP_APV(c0c0c1a0,ed2dd9d0,c74c6690,c0b46172,19b,...) at VOP_LOOKUP_APV+0xe5 lookup(ed2ddb7c,c0b46172,c6,bf,c703a72c,...) at lookup+0x59e namei(ed2ddb7c,c74c6690,ed2dda74,246,c0c61ab8,...) at namei+0x35e vn_open_cred(ed2ddb7c,ed2ddc78,1a4,c75dae00,c74871c8,...) at vn_open_cred+0x62 vn_open(ed2ddb7c,ed2ddc78,1a4,c74871c8,0,...) at vn_open+0x33 kern_open(c74c6690,8164100,0,a02,1a4,...) at kern_open+0xe7 open(c74c6690,ed2ddcfc,c,c0b40d50,c0beb278,...) at open+0x30 syscall(ed2ddd38) at syscall+0x2d3 Xint0x80_syscall() at Xint0x80_syscall+0x20 --- syscall (5, FreeBSD ELF32, open), eip = 0x281a161b, esp = 0xbfbfe8cc, ebp = 0xbfbfe8f8 --- db> trace 55 Tracing pid 55 tid 100054 td 0xc6fea460 sched_switch(c6fea460,0,1,17c,29bc06da,...) at sched_switch+0x49b mi_switch(1,0,c0b3ec35,1cb,c6fea460,...) at mi_switch+0x223 sleepq_switch(c6fea460,0,c0b3ec35,243,68,...) at sleepq_switch+0x182 sleepq_wait(c0c5666c,0,c0b3ba36,d8,0,...) at sleepq_wait+0x60 _sleep(c0c5666c,c0ca8988,68,c0b47461,0,...) at _sleep+0x369 sched_sync(0,eaff0d38,c0b37584,31c,c6febae0,...) at sched_sync+0x752 fork_exit(c0814220,0,eaff0d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xeaff0d70, ebp = 0 --- db> trace 4 Tracing pid 4 tid 100015 td 0xc6797460 sched_switch(c6797460,0,1,17c,5512f4a0,...) at sched_switch+0x49b mi_switch(1,0,c0b3ec35,1cb,4c,...) at mi_switch+0x223 sleepq_switch(c6797460,0,c0b3ec35,266,4c,...) at sleepq_switch+0x182 sleepq_timedwait(c0c5420c,64,c0b32247,0,0,...) at sleepq_timedwait+0x68 _sleep(c0c5420c,c0c54128,24c,c0b32247,64,...) at _sleep+0x345 g_io_schedule_down(c6797460,0,c0b339ab,74,0,...) at g_io_schedule_down+0x6b g_down_procbody(0,e6e39d38,c0b37584,31c,c67eb828,...) at g_down_procbody+0x8d fork_exit(c073d540,0,e6e39d38) at fork_exit+0xb8 fork_trampoline() at fork_trampoline+0x8 --- trap 0, eip = 0, esp = 0xe6e39d70, ebp = 0 --- db> continue ::Uptime: 7m21s ssttPhysical memory: 3310 MB Dumping 225 MB:aacckk bbaacckkttrraaccee:: ddbb__ttrraaccee__sseellff__wwrrappppeerr((cc00bb33ee003388,,eeadf3e4a8ab400c,,cc00881100333333,,00,,eeadf3e4a8ab400c,,......)) aatt ddbb__ttrraaccee__sseellff__wwrraappppeerr++00xx2266 kkddbb__bbaacckkttrraaccee((00,,ee 4a8ab400c,,cc00aaaacc99bb55,,eeadf3e4a8ab510c,, c b4623806dec00,,......)) aatt kkddbb__bbaacckkttrraaccee++00xx2299 vv f bbaaddlloocckk((c00cc00cc11aa00,,eeadf3e4a8ab510c,,cc00cc223388aa00,,ccbb4623806dc00,,00)) aatt vvffss__bbaaddlloocckk++00xx2233 aasssseerrtt__vvoopp__lloocckkeedd((ccbb4623806dec00,,cc ) aatt aasssseerrtt__vvoopp__lloocckkeedd++00xx5500 vvoopp__lloocckk__ppoosstt((eeda3f4e8aba7a8c,,00,,11020,21,21,0c0b242c8b66e300,d.c.0.,). .a.t) vaotp _vloopc_kl_opcoks_tp+o0sxt3+60 x P O _VLOOPC_KL1O_CAKP1V_(AcP0Vc(0cc01ca00c,1eaa0f,eeada3a48,bc708b,4c709bb427,93b222,,3c2b24,2c8b76b340,e.9.4.,). .a.t) VaOtP _VLOOPC_KL1O_CAKP1V_x0PxVc+10 nc_1v __lvonc_kl(occbk4(2c8b66e300,d1c20,,c160f0e2a, c 05,cc306b9404,7cc04b,487a93b,2.,.2.4)2 ,a.t. ._)v na_tl o_ + lxofc8k h0fxlfu8s 0bvunf_qwureiutees((cc760f5e2c62d48,,e0d,3c40bc46407,cc47,58d0abe,0604,,0.,.c.7)5 ca3t6 9f0l,u.s.h.b)u faqtu uvens_+w0rxi5t6e8+ dxb1u6fa_ adeomfoinl(e0w,reiatfee(aedd3384,8cc06b03,7f5f8f4f,f3f1fcf,,cf6fffefcf2bf8f,,.0.,.c)7 0a5t2 6bdu4f,_.d.a.e)m oant+ 0dxocf0 i efworrikt_ee+x0ixfr843_0w,r0i,teeavf(eca7d53c83)6 9a0t, 4f,oerdk3_4e8xci6t0+,08x1b281 o0f0o,r1k9_at,r.a.m.p)o laitn ek(e)r na_tw rfiotrekv_+t0rxa5m8p 5lwirniet+e0(xc87 4c-3-6-9 0t,readp3 408,c feci,pc ,=c 00b,4 0edscp5 ,=c 00bxeeba2f6e0a,d.7.0.,) eabtp w=r i0t e-+-0-x OfV tPs_yLsOcCaKl:l (0exdc3b4482d8368e)0 aits snyostc allolc+k0exd2 db3u y Xsihnotu0lxd8 0b_es sKc + aBl:l (e)n taetr :X ilnotc0kx 8v0i_oslyastciaolnl [thread pid 53 tid 100052 ] Stopped at kdb_enter_why+0x3a: movl $0,kdb_why db> xD2B0: s-t-a-c ks ybsaccakltlr a(c4:, Fdrbe_etBrSaDc eE_LsFe3l2f,_ wwrraiptpee)r,( c0ibp3 e=0 308x,2e8a1fbe1a793e3c,, ce0s8p1 0=3 303x,b0f,befaef9e2ac9,e ce,b.p. .=) 0axtb fdbbf_etlr7a8ce-_-s-e 6fV_OwPr_aLpOpCeKr:+ 00xx2c6b ,3k0ddbc_0baicsk tnroatc el(o0c,keeadf ebau9te cs,hco0ualadc 9bbe5 deKaDfBe:a 9efnct,ecrb:4 2l8o6cek ,v.i.o.l)a taito nk [thread pid 766 tid 100137 ] Stopped at kdb_enter_why+0x3a: movl $0,kdb_why db> r_DbBa:c ksttraaccke +b0axc2k9t bavcfes:_ addbl_otcrka(cce0_cs0ecl1f c _w,reaapfpeear9(fcc0,bc30ec023388,ae0d,3c4b84a2d8c6,ec00,801)0 3a3t3 ,vcf7s5_cb3a6d9l0o,cekd+304x82a3d ,,a.s.s.e)r ta_tv odpb__ltorcakceed_(scebl4f2_8w6rea0p,pce0rb+60fx22465 cck0dbb3_9bca4c9k,tdraa9c3e3(0c1705,c43,6.9.0.,)e da3t4 8aasdsce,rct0_avaocp9_bl5o,cekde3d4+80axe5c0, bV6O3P0_dScT0R,A.T.E.G)Y _aAtP Vk(dcb0_c 0 , c1kat0r,aecaef+e0axa239c cvbf4 ) 2_b7abd4l,occ0kc(2c30cca00c,1cab04,2e8d63e408,a.e.c.,)c 0act2 3V8OaP0_,ScTbR6A3T0EdGcY0_,AcP7V5+c03x699b0 bautf svtfrsa_tbeag , yl(occbk4+20x72e3c daas9s3e3r0t1_0v , c 0_0e,lco0cckae8d6(3c0b,6d3a09d3c300,7c00,b.6.f.4)8 0a,te db3u4f8sbt2rca,ce0g8y1+20dx6601, e0bcu0fcw1rai0t,e.(.d.a)9 3a3t0 1a0s,seearfte_avao9p0_,ecl0oacakfecde+10,xe5a3f aV OaPc_,W0R,I.T.E._)A PaVt( c 0ucf0wcr1iat0e,+e0dx314981b 4v,fcs7_5bci3o6_9a0w,rci0tbe4(7d9ab923,320420,,.0.,.c)0 ba4t4 7VcO4P,_8WaR4I,T8Ea3A,P.V.+.0)x 9a8t 3vvfns__wbriiot_ea(wcr7i0t5e2+60dx43,2e8d 48lcu6s0h,bcu7f5qduaeeu0e0s,(0c,6cf7e5cc23b689,00,,. . 4a7tc 4v,n8_0wbr, t 6t4e,+.0.x.2)3 1a f dfolfuislhebwurfiqtuee(ueeds3+408xc5690c, fbfufff_fd f b fonf(f0f,fefaff,e0a,dc3780,5c206bd347,5.8.4.,)3 1act, cd6offeicl2ebw8r,i.t.e.+)0 xa9t5 5ukfe_rdna_ewmroint+e0vx(c c07 02cf3o6r9k0_,e4x,ietd(3c4087cf6f08,3801,201,4e0a0f,e1a9da3,8.). .a)t afto rkke_renx_iwtr+i0txebv8+ kxf5o8r 0_wtrriatmep(ocl7i5nce3(6)9 0a,te df3o4r8kc_ftcr,acm,pco0lbi4n0ed+c05x,8c =b-e-b-2 6t0r,a.p. .0), aeti pw r=i t0e,+ 0exs4pf c s0yxsecaaflela(d1e7d03,04 8edb3p8 )= a0t -s-y-s EaVlOlP+_0SxT2RdA3T 0GXYi:n t00xxc8b04_2s8y6sec0a lils( )n oatt lXoicnkte0dx 8b0u_ts ysshcoaullld+ 0bxe2 K LD-B-:- esnytsecra:l ll o(c4k, vFiroeleaBtSiDo nE [thread pid 53 tid 100052 ] Stopped at kdb_enter_why+0x3a: movl $0,kdb_why db> panic [no answer... let's power cycle the server once more]