boltdb更新在提交时卡住了。

huangapple go评论89阅读模式
英文:

boltdb update freezes on commit

问题

我想从一个API中提取数据并将其保存到boltDB数据库中。问题是在commit()调用时进程被冻结,没有发送任何panic或错误信息...

有人看到下面的代码有什么问题吗?我一次最多放了1500个蜡烛,也许太多了?如果是这样,对于这么多数据量,最佳实践是什么?

非常感谢!

type Candle struct {
	Close      float32   `json:"close"`
	High       float32   `json:"high"`
	Low        float32   `json:"low"`
	Open       float32   `json:"open"`
	ClockTime  float32   `json:"time"`
	StartTime  time.Time `json:"startTime"`
	Volume     float32   `json:"volume"`
	Resolution string
}

func (s historyDataStore) saveCandles(mkt string, candles []Candle) error {
	tx, err := s.db.Begin(true)
	b := tx.Bucket([]byte(mainBucket)).Bucket([]byte(mkt))
	i := 0
	for _, candle := range candles {
		i++
		js, err := json.Marshal(candle)
		if err != nil {
			return err
		}
		key, err := b.NextSequence()
		err = b.Put(itob(key), js)
		if err != nil {
			return err
		}
	}
	err = tx.Commit()
	if err != nil {
		log.Fatal(err)
	}
	return err
}

这是SIGQUIT信息:
似乎是rwmutex.go中的Lock()函数正在等待其可用性...

PC=0x7fff6b69b882 m=0 sigcode=0

goroutine 0 [idle]:
runtime.pthread_cond_wait(0x1572820, 0x15727e0, 0x0)
        /usr/local/go/src/runtime/sys_darwin.go:384 +0x39
runtime.semasleep(0xffffffffffffffff, 0x7ffeefbff4e0)
        /usr/local/go/src/runtime/os_darwin.go:63 +0x8d
runtime.notesleep(0x15725d0)
        /usr/local/go/src/runtime/lock_sema.go:181 +0xdb
runtime.mPark()
        /usr/local/go/src/runtime/proc.go:1340 +0x39
runtime.stopm()
        /usr/local/go/src/runtime/proc.go:2301 +0x92
runtime.findrunnable(0xc00003b000, 0x0)
        /usr/local/go/src/runtime/proc.go:2960 +0x72e
runtime.schedule()
        /usr/local/go/src/runtime/proc.go:3169 +0x2d7
runtime.park_m(0xc000082900)
        /usr/local/go/src/runtime/proc.go:3318 +0x9d
runtime.mcall(0x106fcf6)
        /usr/local/go/src/runtime/asm_amd64.s:327 +0x5b

goroutine 1 [semacquire, 3 minutes]:
sync.runtime_SemacquireMutex(0xc0001201a0, 0xc0003ece00, 0x0)
        /usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).Lock(0xc000120198)
        /usr/local/go/src/sync/rwmutex.go:116 +0x85
github.com/boltdb/bolt.(*DB).mmap(0xc000120000, 0x9000, 0x0, 0x0)
        /Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/db.go:246 +0x69
github.com/boltdb/bolt.(*DB).allocate(0xc000120000, 0x1, 0xc0003ecfe0, 0x2, 0x2)
        /Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/db.go:849 +0x12a
github.com/boltdb/bolt.(*Tx).allocate(0xc000122000, 0x1, 0x8, 0x8, 0xc0002840a8)
        /Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/tx.go:465 +0xef
github.com/boltdb/bolt.(*node).spill(0xc0002bb110, 0xc0000ba008, 0xc0003ed240)
        /Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/node.go:368 +0x1bd
github.com/boltdb/bolt.(*Bucket).spill(0xc0002d2080, 0xc00034e100, 0xc0003ed4f0)
        /Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/bucket.go:571 +0x5e5
github.com/boltdb/bolt.(*Bucket).spill(0xc0002d2000, 0xc00034e000, 0xc0003ed720)
        /Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/bucket.go:538 +0x496
github.com/boltdb/bolt.(*Bucket).spill(0xc000122018, 0xc0000ba008, 0xc0003ed8a8)
        /Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/bucket.go:538 +0x496
github.com/boltdb/bolt.(*Tx).Commit(0xc000122000, 0x5, 0xc0003edad0)
        /Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/tx.go:165 +0x285
github.com/elRomano/gotrader/store/boltdb.historyDataStore.saveCandles(0xc000120000, 0x13690ca, 0xb, 0x1, 0x1367a43, 0x7, 0xc00075a000, 0x5dd, 0x638, 0x60d34ff4, ...)
        /Users/user/Documents/_DEV/Go/test/store/boltdb/boltStore.go:123 +0x539
github.com/elRomano/gotrader/store/boltdb.historyDataStore.updateMarketHistory(0xc000120000, 0x13690ca, 0xb, 0x5, 0x1367a43, 0x7, 0x12fc9c0, 0x13c7f00)
        /Users/user/Documents/_DEV/Go/test/store/boltdb/boltStore.go:71 +0x3ee
github.com/elRomano/gotrader/store/boltdb.(*historyDataStore).UpdateDb(0xc0000be4c0, 0xc0000be4a0, 0x2, 0x2, 0x1, 0x0)
        /Users/user/Documents/_DEV/Go/test/store/boltdb/boltStore.go:43 +0xa5
main.main()
        /Users/user/Documents/_DEV/Go/test/main.go:59 +0xa0d

rax    0x104
rbx    0x2
rcx    0x7ffeefbff2e8
rdx    0x6c00
rdi    0x1572820
rsi    0x6c0100006d00
rbp    0x7ffeefbff380
rsp    0x7ffeefbff2e8
r8     0x0
r9     0xa0
r10    0x0
r11    0x202
r12    0x1572820
r13    0x16
r14    0x6c0100006d00
r15    0xef99dc0
rip    0x7fff6b69b882
rflags 0x203
cs     0x7
fs     0x0
gs     0x0
exit status 2
英文:

I want to extract data from an API and save it to a boltDB database. The problem is that the process is frozen on the commit() call and sends no panic or error...

Anyone seeing something with the code below ? I put a maximum of 1500 candles at once maybe it's too many ? If so what would be the best practice for such amount ?

Thanks a lot !

type Candle struct {
	Close      float32   `json:"close"`
	High       float32   `json:"high"`
	Low        float32   `json:"low"`
	Open       float32   `json:"open"`
	ClockTime  float32   `json:"time"`
	StartTime  time.Time `json:"startTime"`
	Volume     float32   `json:"volume"`
	Resolution string
}

func (s historyDataStore) saveCandles(mkt string, candles []Candle) error {
	tx, err := s.db.Begin(true)
	b := tx.Bucket([]byte(mainBucket)).Bucket([]byte(mkt))
	i := 0
	for _, candle := range candles {
		i++
		js, err := json.Marshal(candle)
		if err != nil {
			return err
		}
		key, err := b.NextSequence()
		err = b.Put(itob(key), js)
		if err != nil {
			return err
		}
	}
	err = tx.Commit()
	if err != nil {
		log.Fatal(err)
	}
	return err
}

This is the SIGQUIT info :
It seems that this is the Lock() function in rwmutex.go that is waiting for its availability...

PC=0x7fff6b69b882 m=0 sigcode=0

goroutine 0 [idle]:
runtime.pthread_cond_wait(0x1572820, 0x15727e0, 0x0)
        /usr/local/go/src/runtime/sys_darwin.go:384 +0x39
runtime.semasleep(0xffffffffffffffff, 0x7ffeefbff4e0)
        /usr/local/go/src/runtime/os_darwin.go:63 +0x8d
runtime.notesleep(0x15725d0)
        /usr/local/go/src/runtime/lock_sema.go:181 +0xdb
runtime.mPark()
        /usr/local/go/src/runtime/proc.go:1340 +0x39
runtime.stopm()
        /usr/local/go/src/runtime/proc.go:2301 +0x92
runtime.findrunnable(0xc00003b000, 0x0)
        /usr/local/go/src/runtime/proc.go:2960 +0x72e
runtime.schedule()
        /usr/local/go/src/runtime/proc.go:3169 +0x2d7
runtime.park_m(0xc000082900)
        /usr/local/go/src/runtime/proc.go:3318 +0x9d
runtime.mcall(0x106fcf6)
        /usr/local/go/src/runtime/asm_amd64.s:327 +0x5b

goroutine 1 [semacquire, 3 minutes]:
sync.runtime_SemacquireMutex(0xc0001201a0, 0xc0003ece00, 0x0)
        /usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).Lock(0xc000120198)
        /usr/local/go/src/sync/rwmutex.go:116 +0x85
github.com/boltdb/bolt.(*DB).mmap(0xc000120000, 0x9000, 0x0, 0x0)
        /Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/db.go:246 +0x69
github.com/boltdb/bolt.(*DB).allocate(0xc000120000, 0x1, 0xc0003ecfe0, 0x2, 0x2)
        /Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/db.go:849 +0x12a
github.com/boltdb/bolt.(*Tx).allocate(0xc000122000, 0x1, 0x8, 0x8, 0xc0002840a8)
        /Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/tx.go:465 +0xef
github.com/boltdb/bolt.(*node).spill(0xc0002bb110, 0xc0000ba008, 0xc0003ed240)
        /Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/node.go:368 +0x1bd
github.com/boltdb/bolt.(*Bucket).spill(0xc0002d2080, 0xc00034e100, 0xc0003ed4f0)
        /Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/bucket.go:571 +0x5e5
github.com/boltdb/bolt.(*Bucket).spill(0xc0002d2000, 0xc00034e000, 0xc0003ed720)
        /Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/bucket.go:538 +0x496
github.com/boltdb/bolt.(*Bucket).spill(0xc000122018, 0xc0000ba008, 0xc0003ed8a8)
        /Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/bucket.go:538 +0x496
github.com/boltdb/bolt.(*Tx).Commit(0xc000122000, 0x5, 0xc0003edad0)
        /Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/tx.go:165 +0x285
github.com/elRomano/gotrader/store/boltdb.historyDataStore.saveCandles(0xc000120000, 0x13690ca, 0xb, 0x1, 0x1367a43, 0x7, 0xc00075a000, 0x5dd, 0x638, 0x60d34ff4, ...)
        /Users/user/Documents/_DEV/Go/test/store/boltdb/boltStore.go:123 +0x539
github.com/elRomano/gotrader/store/boltdb.historyDataStore.updateMarketHistory(0xc000120000, 0x13690ca, 0xb, 0x5, 0x1367a43, 0x7, 0x12fc9c0, 0x13c7f00)
        /Users/user/Documents/_DEV/Go/test/store/boltdb/boltStore.go:71 +0x3ee
github.com/elRomano/gotrader/store/boltdb.(*historyDataStore).UpdateDb(0xc0000be4c0, 0xc0000be4a0, 0x2, 0x2, 0x1, 0x0)
        /Users/user/Documents/_DEV/Go/test/store/boltdb/boltStore.go:43 +0xa5
main.main()
        /Users/user/Documents/_DEV/Go/test/main.go:59 +0xa0d

rax    0x104
rbx    0x2
rcx    0x7ffeefbff2e8
rdx    0x6c00
rdi    0x1572820
rsi    0x6c0100006d00
rbp    0x7ffeefbff380
rsp    0x7ffeefbff2e8
r8     0x0
r9     0xa0
r10    0x0
r11    0x202
r12    0x1572820
r13    0x16
r14    0x6c0100006d00
r15    0xef99dc0
rip    0x7fff6b69b882
rflags 0x203
cs     0x7
fs     0x0
gs     0x0
exit status 2

答案1

得分: 0

感谢SIGQUIT跟踪,我发现有一个“Begin”事务被打开,它锁定了我想执行的其他事务。我在最后添加了一个回滚操作,然后问题就解决了。

问题解决了!谢谢@rustyx @mh-cbon @Adrian。

英文:

Thanks to SIGQUIT tracing I've found out that there was a "Begin" transaction opened that was locking the other transactions I wanted to perform. I added a Rollback at the end and then it worked.

Problem solved ! Thanks @rustyx @mh-cbon @Adrian.

huangapple
  • 本文由 发表于 2021年6月24日 23:25:00
  • 转载请务必保留本文链接:https://go.coder-hub.com/68118584.html
匿名

发表评论

匿名网友

:?: :razz: :sad: :evil: :!: :smile: :oops: :grin: :eek: :shock: :???: :cool: :lol: :mad: :twisted: :roll: :wink: :idea: :arrow: :neutral: :cry: :mrgreen:

确定