英文:
boltdb update freezes on commit
问题
我想从一个API中提取数据并将其保存到boltDB数据库中。问题是在commit()调用时进程被冻结,没有发送任何panic或错误信息...
有人看到下面的代码有什么问题吗?我一次最多放了1500个蜡烛,也许太多了?如果是这样,对于这么多数据量,最佳实践是什么?
非常感谢!
type Candle struct {
Close float32 `json:"close"`
High float32 `json:"high"`
Low float32 `json:"low"`
Open float32 `json:"open"`
ClockTime float32 `json:"time"`
StartTime time.Time `json:"startTime"`
Volume float32 `json:"volume"`
Resolution string
}
func (s historyDataStore) saveCandles(mkt string, candles []Candle) error {
tx, err := s.db.Begin(true)
b := tx.Bucket([]byte(mainBucket)).Bucket([]byte(mkt))
i := 0
for _, candle := range candles {
i++
js, err := json.Marshal(candle)
if err != nil {
return err
}
key, err := b.NextSequence()
err = b.Put(itob(key), js)
if err != nil {
return err
}
}
err = tx.Commit()
if err != nil {
log.Fatal(err)
}
return err
}
这是SIGQUIT信息:
似乎是rwmutex.go中的Lock()函数正在等待其可用性...
PC=0x7fff6b69b882 m=0 sigcode=0
goroutine 0 [idle]:
runtime.pthread_cond_wait(0x1572820, 0x15727e0, 0x0)
/usr/local/go/src/runtime/sys_darwin.go:384 +0x39
runtime.semasleep(0xffffffffffffffff, 0x7ffeefbff4e0)
/usr/local/go/src/runtime/os_darwin.go:63 +0x8d
runtime.notesleep(0x15725d0)
/usr/local/go/src/runtime/lock_sema.go:181 +0xdb
runtime.mPark()
/usr/local/go/src/runtime/proc.go:1340 +0x39
runtime.stopm()
/usr/local/go/src/runtime/proc.go:2301 +0x92
runtime.findrunnable(0xc00003b000, 0x0)
/usr/local/go/src/runtime/proc.go:2960 +0x72e
runtime.schedule()
/usr/local/go/src/runtime/proc.go:3169 +0x2d7
runtime.park_m(0xc000082900)
/usr/local/go/src/runtime/proc.go:3318 +0x9d
runtime.mcall(0x106fcf6)
/usr/local/go/src/runtime/asm_amd64.s:327 +0x5b
goroutine 1 [semacquire, 3 minutes]:
sync.runtime_SemacquireMutex(0xc0001201a0, 0xc0003ece00, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).Lock(0xc000120198)
/usr/local/go/src/sync/rwmutex.go:116 +0x85
github.com/boltdb/bolt.(*DB).mmap(0xc000120000, 0x9000, 0x0, 0x0)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/db.go:246 +0x69
github.com/boltdb/bolt.(*DB).allocate(0xc000120000, 0x1, 0xc0003ecfe0, 0x2, 0x2)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/db.go:849 +0x12a
github.com/boltdb/bolt.(*Tx).allocate(0xc000122000, 0x1, 0x8, 0x8, 0xc0002840a8)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/tx.go:465 +0xef
github.com/boltdb/bolt.(*node).spill(0xc0002bb110, 0xc0000ba008, 0xc0003ed240)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/node.go:368 +0x1bd
github.com/boltdb/bolt.(*Bucket).spill(0xc0002d2080, 0xc00034e100, 0xc0003ed4f0)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/bucket.go:571 +0x5e5
github.com/boltdb/bolt.(*Bucket).spill(0xc0002d2000, 0xc00034e000, 0xc0003ed720)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/bucket.go:538 +0x496
github.com/boltdb/bolt.(*Bucket).spill(0xc000122018, 0xc0000ba008, 0xc0003ed8a8)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/bucket.go:538 +0x496
github.com/boltdb/bolt.(*Tx).Commit(0xc000122000, 0x5, 0xc0003edad0)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/tx.go:165 +0x285
github.com/elRomano/gotrader/store/boltdb.historyDataStore.saveCandles(0xc000120000, 0x13690ca, 0xb, 0x1, 0x1367a43, 0x7, 0xc00075a000, 0x5dd, 0x638, 0x60d34ff4, ...)
/Users/user/Documents/_DEV/Go/test/store/boltdb/boltStore.go:123 +0x539
github.com/elRomano/gotrader/store/boltdb.historyDataStore.updateMarketHistory(0xc000120000, 0x13690ca, 0xb, 0x5, 0x1367a43, 0x7, 0x12fc9c0, 0x13c7f00)
/Users/user/Documents/_DEV/Go/test/store/boltdb/boltStore.go:71 +0x3ee
github.com/elRomano/gotrader/store/boltdb.(*historyDataStore).UpdateDb(0xc0000be4c0, 0xc0000be4a0, 0x2, 0x2, 0x1, 0x0)
/Users/user/Documents/_DEV/Go/test/store/boltdb/boltStore.go:43 +0xa5
main.main()
/Users/user/Documents/_DEV/Go/test/main.go:59 +0xa0d
rax 0x104
rbx 0x2
rcx 0x7ffeefbff2e8
rdx 0x6c00
rdi 0x1572820
rsi 0x6c0100006d00
rbp 0x7ffeefbff380
rsp 0x7ffeefbff2e8
r8 0x0
r9 0xa0
r10 0x0
r11 0x202
r12 0x1572820
r13 0x16
r14 0x6c0100006d00
r15 0xef99dc0
rip 0x7fff6b69b882
rflags 0x203
cs 0x7
fs 0x0
gs 0x0
exit status 2
英文:
I want to extract data from an API and save it to a boltDB database. The problem is that the process is frozen on the commit() call and sends no panic or error...
Anyone seeing something with the code below ? I put a maximum of 1500 candles at once maybe it's too many ? If so what would be the best practice for such amount ?
Thanks a lot !
type Candle struct {
Close float32 `json:"close"`
High float32 `json:"high"`
Low float32 `json:"low"`
Open float32 `json:"open"`
ClockTime float32 `json:"time"`
StartTime time.Time `json:"startTime"`
Volume float32 `json:"volume"`
Resolution string
}
func (s historyDataStore) saveCandles(mkt string, candles []Candle) error {
tx, err := s.db.Begin(true)
b := tx.Bucket([]byte(mainBucket)).Bucket([]byte(mkt))
i := 0
for _, candle := range candles {
i++
js, err := json.Marshal(candle)
if err != nil {
return err
}
key, err := b.NextSequence()
err = b.Put(itob(key), js)
if err != nil {
return err
}
}
err = tx.Commit()
if err != nil {
log.Fatal(err)
}
return err
}
This is the SIGQUIT info :
It seems that this is the Lock() function in rwmutex.go that is waiting for its availability...
PC=0x7fff6b69b882 m=0 sigcode=0
goroutine 0 [idle]:
runtime.pthread_cond_wait(0x1572820, 0x15727e0, 0x0)
/usr/local/go/src/runtime/sys_darwin.go:384 +0x39
runtime.semasleep(0xffffffffffffffff, 0x7ffeefbff4e0)
/usr/local/go/src/runtime/os_darwin.go:63 +0x8d
runtime.notesleep(0x15725d0)
/usr/local/go/src/runtime/lock_sema.go:181 +0xdb
runtime.mPark()
/usr/local/go/src/runtime/proc.go:1340 +0x39
runtime.stopm()
/usr/local/go/src/runtime/proc.go:2301 +0x92
runtime.findrunnable(0xc00003b000, 0x0)
/usr/local/go/src/runtime/proc.go:2960 +0x72e
runtime.schedule()
/usr/local/go/src/runtime/proc.go:3169 +0x2d7
runtime.park_m(0xc000082900)
/usr/local/go/src/runtime/proc.go:3318 +0x9d
runtime.mcall(0x106fcf6)
/usr/local/go/src/runtime/asm_amd64.s:327 +0x5b
goroutine 1 [semacquire, 3 minutes]:
sync.runtime_SemacquireMutex(0xc0001201a0, 0xc0003ece00, 0x0)
/usr/local/go/src/runtime/sema.go:71 +0x47
sync.(*RWMutex).Lock(0xc000120198)
/usr/local/go/src/sync/rwmutex.go:116 +0x85
github.com/boltdb/bolt.(*DB).mmap(0xc000120000, 0x9000, 0x0, 0x0)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/db.go:246 +0x69
github.com/boltdb/bolt.(*DB).allocate(0xc000120000, 0x1, 0xc0003ecfe0, 0x2, 0x2)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/db.go:849 +0x12a
github.com/boltdb/bolt.(*Tx).allocate(0xc000122000, 0x1, 0x8, 0x8, 0xc0002840a8)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/tx.go:465 +0xef
github.com/boltdb/bolt.(*node).spill(0xc0002bb110, 0xc0000ba008, 0xc0003ed240)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/node.go:368 +0x1bd
github.com/boltdb/bolt.(*Bucket).spill(0xc0002d2080, 0xc00034e100, 0xc0003ed4f0)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/bucket.go:571 +0x5e5
github.com/boltdb/bolt.(*Bucket).spill(0xc0002d2000, 0xc00034e000, 0xc0003ed720)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/bucket.go:538 +0x496
github.com/boltdb/bolt.(*Bucket).spill(0xc000122018, 0xc0000ba008, 0xc0003ed8a8)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/bucket.go:538 +0x496
github.com/boltdb/bolt.(*Tx).Commit(0xc000122000, 0x5, 0xc0003edad0)
/Users/user/go/pkg/mod/github.com/boltdb/bolt@v1.3.1/tx.go:165 +0x285
github.com/elRomano/gotrader/store/boltdb.historyDataStore.saveCandles(0xc000120000, 0x13690ca, 0xb, 0x1, 0x1367a43, 0x7, 0xc00075a000, 0x5dd, 0x638, 0x60d34ff4, ...)
/Users/user/Documents/_DEV/Go/test/store/boltdb/boltStore.go:123 +0x539
github.com/elRomano/gotrader/store/boltdb.historyDataStore.updateMarketHistory(0xc000120000, 0x13690ca, 0xb, 0x5, 0x1367a43, 0x7, 0x12fc9c0, 0x13c7f00)
/Users/user/Documents/_DEV/Go/test/store/boltdb/boltStore.go:71 +0x3ee
github.com/elRomano/gotrader/store/boltdb.(*historyDataStore).UpdateDb(0xc0000be4c0, 0xc0000be4a0, 0x2, 0x2, 0x1, 0x0)
/Users/user/Documents/_DEV/Go/test/store/boltdb/boltStore.go:43 +0xa5
main.main()
/Users/user/Documents/_DEV/Go/test/main.go:59 +0xa0d
rax 0x104
rbx 0x2
rcx 0x7ffeefbff2e8
rdx 0x6c00
rdi 0x1572820
rsi 0x6c0100006d00
rbp 0x7ffeefbff380
rsp 0x7ffeefbff2e8
r8 0x0
r9 0xa0
r10 0x0
r11 0x202
r12 0x1572820
r13 0x16
r14 0x6c0100006d00
r15 0xef99dc0
rip 0x7fff6b69b882
rflags 0x203
cs 0x7
fs 0x0
gs 0x0
exit status 2
答案1
得分: 0
感谢SIGQUIT跟踪,我发现有一个“Begin”事务被打开,它锁定了我想执行的其他事务。我在最后添加了一个回滚操作,然后问题就解决了。
问题解决了!谢谢@rustyx @mh-cbon @Adrian。
英文:
Thanks to SIGQUIT tracing I've found out that there was a "Begin" transaction opened that was locking the other transactions I wanted to perform. I added a Rollback at the end and then it worked.
Problem solved ! Thanks @rustyx @mh-cbon @Adrian.
通过集体智慧和协作来改善编程学习和解决问题的方式。致力于成为全球开发者共同参与的知识库,让每个人都能够通过互相帮助和分享经验来进步。
评论