shkong/go-cqhttp
1
0
Fork 0
mirror of https://github.com/Mrs4s/go-cqhttp.git synced 2025-05-05 03:23:49 +08:00
Code Issues Projects Releases Wiki Activity

feat(internal/btree): use md5 key instead of sha1

Browse Source 
image/video cache uses md5 key, this commit will reduce 4 used bytes in db
This commit is contained in:
wdvxdr 2021-10-03 22:32:11 +08:00
parent ce6b65ddb5
commit 63e950bb83
No known key found for this signature in database
GPG Key ID: 703F8C071DE7A1B6
4 changed files with 66 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
internal/btree/btree.go
View File

@ -11,7 +11,7 @@ import (
) )
const ( const (
sha1Size = 20 // md5 sha1 hashSize = 16 // md5 hash
tableSize = (1024 - 1) / int(unsafe.Sizeof(item{})) tableSize = (1024 - 1) / int(unsafe.Sizeof(item{}))
cacheSlots = 11 // prime cacheSlots = 11 // prime
superSize = int(unsafe.Sizeof(super{})) superSize = int(unsafe.Sizeof(super{}))
@ -19,7 +19,7 @@ const (
) )
type item struct { type item struct {
sha1 [sha1Size]byte hash [hashSize]byte
offset int64 offset int64
child int64 child int64
} }
@ -46,10 +46,12 @@ type DB struct {
top int64 top int64
freeTop int64 freeTop int64
alloc int64 alloc int64
cache [23]cache cache [cacheSlots]cache
inAllocator bool inAllocator bool
deleteLarger bool deleteLarger bool
fqueue [freeQueueLen]chunk
fqueueLen int
} }
func (d *DB) get(offset int64) *table { func (d *DB) get(offset int64) *table {
@ -163,10 +165,10 @@ func collapse(bt *DB, offset int64) int64 {
return ret return ret
} }
// split a table. The pivot item is stored to 'sha1' and 'offset'. // split a table. The pivot item is stored to 'hash' and 'offset'.
// Returns offset to the new table. // Returns offset to the new table.
func (d *DB) split(t *table, hash *byte, offset *int64) int64 { func (d *DB) split(t *table, hash *byte, offset *int64) int64 {
copysha1(hash, &t.items[tableSize/2].sha1[0]) copyhash(hash, &t.items[tableSize/2].hash[0])
*offset = t.items[tableSize/2].offset *offset = t.items[tableSize/2].offset
ntable := new(table) ntable := new(table)
@ -185,7 +187,7 @@ func (d *DB) split(t *table, hash *byte, offset *int64) int64 {
} }
// takeSmallest find and remove the smallest item from the given table. The key of the item // takeSmallest find and remove the smallest item from the given table. The key of the item
// is stored to 'sha1'. Returns offset to the item // is stored to 'hash'. Returns offset to the item
func (d *DB) takeSmallest(toff int64, sha1 *byte) int64 { func (d *DB) takeSmallest(toff int64, sha1 *byte) int64 {
table := d.get(toff) table := d.get(toff)
assert(table.size > 0) assert(table.size > 0)
@ -207,18 +209,18 @@ func (d *DB) takeSmallest(toff int64, sha1 *byte) int64 {
} }
// takeLargest find and remove the largest item from the given table. The key of the item // takeLargest find and remove the largest item from the given table. The key of the item
// is stored to 'sha1'. Returns offset to the item // is stored to 'hash'. Returns offset to the item
func (d *DB) takeLargest(toff int64, sha1 *byte) int64 { func (d *DB) takeLargest(toff int64, hash *byte) int64 {
table := d.get(toff) table := d.get(toff)
assert(table.size > 0) assert(table.size > 0)
var off int64 var off int64
child := table.items[table.size].child child := table.items[table.size].child
if child == 0 { if child == 0 {
off = d.remove(table, table.size-1, sha1) off = d.remove(table, table.size-1, hash)
} else { } else {
/* recursion */ /* recursion */
off = d.takeLargest(child, sha1) off = d.takeLargest(child, hash)
table.items[table.size].child = collapse(d, child) table.items[table.size].child = collapse(d, child)
} }
d.flush(table, toff) d.flush(table, toff)
@ -229,12 +231,12 @@ func (d *DB) takeLargest(toff int64, sha1 *byte) int64 {
} }
// remove an item in position 'i' from the given table. The key of the // remove an item in position 'i' from the given table. The key of the
// removed item is stored to 'sha1'. Returns offset to the item. // removed item is stored to 'hash'. Returns offset to the item.
func (d *DB) remove(t *table, i int, sha1 *byte) int64 { func (d *DB) remove(t *table, i int, hash *byte) int64 {
assert(i < t.size) assert(i < t.size)
if sha1 != nil { if hash != nil {
copysha1(sha1, &t.items[i].sha1[0]) copyhash(hash, &t.items[i].hash[0])
} }
offset := t.items[i].offset offset := t.items[i].offset
@ -246,10 +248,10 @@ func (d *DB) remove(t *table, i int, sha1 *byte) int64 {
child tables */ child tables */
var noff int64 var noff int64
if rand.Int()&1 != 0 { if rand.Int()&1 != 0 {
noff = d.takeLargest(lc, &t.items[i].sha1[0]) noff = d.takeLargest(lc, &t.items[i].hash[0])
t.items[i].child = collapse(d, lc) t.items[i].child = collapse(d, lc)
} else { } else {
noff = d.takeSmallest(rc, &t.items[i].sha1[0]) noff = d.takeSmallest(rc, &t.items[i].hash[0])
t.items[i+1].child = collapse(d, rc) t.items[i+1].child = collapse(d, rc)
} }
t.items[i].child = noff t.items[i].child = noff
@ -268,14 +270,14 @@ func (d *DB) remove(t *table, i int, sha1 *byte) int64 {
return offset return offset
} }
func (d *DB) insert(toff int64, sha1 *byte, data []byte, size int) int64 { func (d *DB) insert(toff int64, hash *byte, data []byte, size int) int64 {
table := d.get(toff) table := d.get(toff)
assert(table.size < tableSize-1) assert(table.size < tableSize-1)
left, right := 0, table.size left, right := 0, table.size
for left < right { for left < right {
mid := (right-left)>>1 + left mid := (right-left)>>1 + left
switch cmp := cmp(sha1, &table.items[mid].sha1[0]); { switch cmp := cmp(hash, &table.items[mid].hash[0]); {
case cmp == 0: case cmp == 0:
// already in the table // already in the table
ret := table.items[mid].offset ret := table.items[mid].offset
@ -293,7 +295,7 @@ func (d *DB) insert(toff int64, sha1 *byte, data []byte, size int) int64 {
lc := table.items[i].child lc := table.items[i].child
if lc != 0 { if lc != 0 {
/* recursion */ /* recursion */
ret = d.insert(lc, sha1, data, size) ret = d.insert(lc, hash, data, size)
/* check if we need to split */ /* check if we need to split */
child := d.get(lc) child := d.get(lc)
@ -304,7 +306,7 @@ func (d *DB) insert(toff int64, sha1 *byte, data []byte, size int) int64 {
return ret return ret
} }
/* overwrites SHA-1 */ /* overwrites SHA-1 */
rc = d.split(child, sha1, &off) rc = d.split(child, hash, &off)
/* flush just in case changes happened */ /* flush just in case changes happened */
d.flush(child, lc) d.flush(child, lc)
@ -319,7 +321,7 @@ func (d *DB) insert(toff int64, sha1 *byte, data []byte, size int) int64 {
// memmove(&table->items[i + 1], &table->items[i], // memmove(&table->items[i + 1], &table->items[i],
// (table->size - i) * sizeof(struct btree_item)); // (table->size - i) * sizeof(struct btree_item));
copy(table.items[i+1:], table.items[i:]) copy(table.items[i+1:], table.items[i:])
copysha1(&table.items[i].sha1[0], sha1) copyhash(&table.items[i].hash[0], hash)
table.items[i].offset = off table.items[i].offset = off
table.items[i].child = lc table.items[i].child = lc
table.items[i+1].child = rc table.items[i+1].child = rc
@ -351,9 +353,9 @@ func (d *DB) insertData(data []byte, size int) int64 {
return offset return offset
} }
// delete remove an item with key 'sha1' from the given table. The offset to the // delete remove an item with key 'hash' from the given table. The offset to the
// removed item is returned. // removed item is returned.
// Please note that 'sha1' is overwritten when called inside the allocator. // Please note that 'hash' is overwritten when called inside the allocator.
func (d *DB) delete(offset int64, hash *byte) int64 { func (d *DB) delete(offset int64, hash *byte) int64 {
if offset == 0 { if offset == 0 {
return 0 return 0
@ -363,7 +365,7 @@ func (d *DB) delete(offset int64, hash *byte) int64 {
left, right := 0, table.size left, right := 0, table.size
for left < right { for left < right {
i := (right-left)>>1 + left i := (right-left)>>1 + left
switch cmp := cmp(hash, &table.items[i].sha1[0]); { switch cmp := cmp(hash, &table.items[i].hash[0]); {
case cmp == 0: case cmp == 0:
// found // found
ret := d.remove(table, i, hash) ret := d.remove(table, i, hash)
@ -396,10 +398,10 @@ func (d *DB) delete(offset int64, hash *byte) int64 {
return ret return ret
} }
func (d *DB) insertTopLevel(toff *int64, sha1 *byte, data []byte, size int) int64 { // nolint:unparam func (d *DB) insertTopLevel(toff *int64, hash *byte, data []byte, size int) int64 { // nolint:unparam
var off, ret, rc int64 var off, ret, rc int64
if *toff != 0 { if *toff != 0 {
ret = d.insert(*toff, sha1, data, size) ret = d.insert(*toff, hash, data, size)
/* check if we need to split */ /* check if we need to split */
table := d.get(*toff) table := d.get(*toff)
@ -408,7 +410,7 @@ func (d *DB) insertTopLevel(toff *int64, sha1 *byte, data []byte, size int) int6
d.put(table, *toff) d.put(table, *toff)
return ret return ret
} }
rc = d.split(table, sha1, &off) rc = d.split(table, hash, &off)
d.flush(table, *toff) d.flush(table, *toff)
} else { } else {
off = d.insertData(data, size) off = d.insertData(data, size)
@ -418,7 +420,7 @@ func (d *DB) insertTopLevel(toff *int64, sha1 *byte, data []byte, size int) int6
/* create new top level table */ /* create new top level table */
t := new(table) t := new(table)
t.size = 1 t.size = 1
copysha1(&t.items[0].sha1[0], sha1) copyhash(&t.items[0].hash[0], hash)
t.items[0].offset = off t.items[0].offset = off
t.items[0].child = *toff t.items[0].child = *toff
t.items[1].child = rc t.items[1].child = rc
@ -433,7 +435,7 @@ func (d *DB) insertTopLevel(toff *int64, sha1 *byte, data []byte, size int) int6
return ret return ret
} }
func (d *DB) lookup(toff int64, sha1 *byte) int64 { func (d *DB) lookup(toff int64, hash *byte) int64 {
if toff == 0 { if toff == 0 {
return 0 return 0
} }
@ -442,7 +444,7 @@ func (d *DB) lookup(toff int64, sha1 *byte) int64 {
left, right := 0, table.size left, right := 0, table.size
for left < right { for left < right {
mid := (right-left)>>1 + left mid := (right-left)>>1 + left
switch cmp := cmp(sha1, &table.items[mid].sha1[0]); { switch cmp := cmp(hash, &table.items[mid].hash[0]); {
case cmp == 0: case cmp == 0:
// found // found
ret := table.items[mid].offset ret := table.items[mid].offset
@ -458,26 +460,26 @@ func (d *DB) lookup(toff int64, sha1 *byte) int64 {
i := left i := left
child := table.items[i].child child := table.items[i].child
d.put(table, toff) d.put(table, toff)
return d.lookup(child, sha1) return d.lookup(child, hash)
} }
// Insert a new item with key 'sha1' with the contents in 'data' to the // Insert a new item with key 'hash' with the contents in 'data' to the
// database file. // database file.
func (d *DB) Insert(csha1 *byte, data []byte) { func (d *DB) Insert(chash *byte, data []byte) {
/* SHA-1 must be in writable memory */ /* SHA-1 must be in writable memory */
var sha1 [sha1Size]byte var hash [hashSize]byte
copysha1(&sha1[0], csha1) copyhash(&hash[0], chash)
_ = d.insertTopLevel(&d.top, &sha1[0], data, len(data)) _ = d.insertTopLevel(&d.top, &hash[0], data, len(data))
freeQueued(d) freeQueued(d)
d.flushSuper() d.flushSuper()
} }
// Get look up item with the given key 'sha1' in the database file. Length of the // Get look up item with the given key 'hash' in the database file. Length of the
// item is stored in 'len'. Returns a pointer to the contents of the item. // item is stored in 'len'. Returns a pointer to the contents of the item.
// The returned pointer should be released with free() after use. // The returned pointer should be released with free() after use.
func (d *DB) Get(sha1 *byte) []byte { func (d *DB) Get(hash *byte) []byte {
off := d.lookup(d.top, sha1) off := d.lookup(d.top, hash)
if off == 0 { if off == 0 {
return nil return nil
} }
@ -495,7 +497,7 @@ func (d *DB) Get(sha1 *byte) []byte {
return data[:n] return data[:n]
} }
// Delete remove item with the given key 'sha1' from the database file. // Delete remove item with the given key 'hash' from the database file.
func (d *DB) Delete(sha1 *byte) error { func (d *DB) Delete(sha1 *byte) error {
return errors.New("impl me") return errors.New("impl me")
} }

38
internal/btree/chunk.go
View File

@ -12,18 +12,12 @@ type chunk struct {
const freeQueueLen = 64 const freeQueueLen = 64
// todo(wdvxdr): move this to btree?
var (
fqueue [freeQueueLen]chunk
fqueueLen = 0
)
func freeQueued(bt *DB) { func freeQueued(bt *DB) {
for i := 0; i < fqueueLen; i++ { for i := 0; i < bt.fqueueLen; i++ {
chunk := &fqueue[i] chunk := &bt.fqueue[i]
bt.freeChunk(chunk.offset, chunk.len) bt.freeChunk(chunk.offset, chunk.len)
} }
fqueueLen = 0 bt.fqueueLen = 0
} }
func (d *DB) allocChunk(size int) int64 { func (d *DB) allocChunk(size int) int64 {
@ -36,10 +30,10 @@ func (d *DB) allocChunk(size int) int64 {
const i32s = unsafe.Sizeof(int32(0)) const i32s = unsafe.Sizeof(int32(0))
/* create fake size SHA-1 */ /* create fake size SHA-1 */
var sha1 [sha1Size]byte var sha1 [hashSize]byte
p := unsafe.Pointer(&sha1[0]) p := unsafe.Pointer(&sha1[0])
*(*int32)(p) = -1 // *(uint32_t *) sha1 = -1; *(*int32)(p) = -1 // *(uint32_t *) hash = -1;
*(*uint32)(unsafe.Add(p, i32s)) = uint32(size) // ((__be32 *) sha1)[1] = to_be32(size); *(*uint32)(unsafe.Add(p, i32s)) = uint32(size) // ((__be32 *) hash)[1] = to_be32(size);
/* find free chunk with the larger or the same size/SHA-1 */ /* find free chunk with the larger or the same size/SHA-1 */
d.inAllocator = true d.inAllocator = true
@ -47,13 +41,13 @@ func (d *DB) allocChunk(size int) int64 {
offset = d.delete(d.freeTop, &sha1[0]) offset = d.delete(d.freeTop, &sha1[0])
d.deleteLarger = false d.deleteLarger = false
if offset != 0 { if offset != 0 {
assert(*(*int32)(p) == -1) // assert(*(uint32_t *) sha1 == (uint32_t) -1) assert(*(*int32)(p) == -1) // assert(*(uint32_t *) hash == (uint32_t) -1)
flen := int(*(*uint32)(unsafe.Add(p, i32s))) // size_t free_len = from_be32(((__be32 *) sha1)[1]) flen := int(*(*uint32)(unsafe.Add(p, i32s))) // size_t free_len = from_be32(((__be32 *) hash)[1])
assert(power2(flen) == flen) assert(power2(flen) == flen)
assert(flen >= size) assert(flen >= size)
/* delete buddy information */ /* delete buddy information */
resetsha1(&sha1[0]) resethash(&sha1[0])
*(*int64)(p) = offset *(*int64)(p) = offset
buddyLen := d.delete(d.freeTop, &sha1[0]) buddyLen := d.delete(d.freeTop, &sha1[0])
assert(buddyLen == int64(size)) assert(buddyLen == int64(size))
@ -95,28 +89,28 @@ func (d *DB) freeChunk(offset int64, size int) {
assert(offset&int64(size-1) == 0) assert(offset&int64(size-1) == 0)
if d.inAllocator { if d.inAllocator {
chunk := &fqueue[fqueueLen] chunk := &d.fqueue[d.fqueueLen]
fqueueLen++ d.fqueueLen++
chunk.offset = offset chunk.offset = offset
chunk.len = size chunk.len = size
return return
} }
/* create fake offset SHA-1 for buddy allocation */ /* create fake offset SHA-1 for buddy allocation */
var sha1 [sha1Size]byte var sha1 [hashSize]byte
p := unsafe.Pointer(&sha1[0]) p := unsafe.Pointer(&sha1[0])
d.inAllocator = true d.inAllocator = true
const i32s = unsafe.Sizeof(int32(0)) const i32s = unsafe.Sizeof(int32(0))
/* add buddy information */ /* add buddy information */
resetsha1(&sha1[0]) resethash(&sha1[0])
*(*int32)(p) = -1 // *(uint32_t *) sha1 = -1; *(*int32)(p) = -1 // *(uint32_t *) hash = -1;
*(*uint32)(unsafe.Add(p, i32s)) = uint32(size) // ((__be32 *) sha1)[1] = to_be32(size); *(*uint32)(unsafe.Add(p, i32s)) = uint32(size) // ((__be32 *) hash)[1] = to_be32(size);
*(*uint32)(unsafe.Add(p, i32s*2)) = rand.Uint32() /* to make SHA-1 unique */ *(*uint32)(unsafe.Add(p, i32s*2)) = rand.Uint32() /* to make SHA-1 unique */
*(*uint32)(unsafe.Add(p, i32s*3)) = rand.Uint32() *(*uint32)(unsafe.Add(p, i32s*3)) = rand.Uint32()
// insert_toplevel(btree, &btree->free_top, sha1, NULL, offset); // insert_toplevel(btree, &btree->free_top, hash, NULL, offset);
_ = d.insertTopLevel(&d.freeTop, &sha1[0], nil, int(offset)) _ = d.insertTopLevel(&d.freeTop, &sha1[0], nil, int(offset))
d.inAllocator = false d.inAllocator = false

15
internal/btree/helper.go
View File

@ -21,7 +21,7 @@ func power2(val int) int {
return i return i
} }
// helpers for sha1 // helpers for hash
func cmp(a, b *byte) int64 { func cmp(a, b *byte) int64 {
pa, pb := unsafe.Pointer(a), unsafe.Pointer(b) pa, pb := unsafe.Pointer(a), unsafe.Pointer(b)
@ -29,20 +29,17 @@ func cmp(a, b *byte) int64 {
return int64(*(*uint64)(pa) - *(*uint64)(pb)) return int64(*(*uint64)(pa) - *(*uint64)(pb))
} }
pa, pb = unsafe.Add(pa, 8), unsafe.Add(pb, 8) pa, pb = unsafe.Add(pa, 8), unsafe.Add(pb, 8)
if *(*uint64)(pa) != *(*uint64)(pb) { return int64(*(*uint64)(pa) - *(*uint64)(pb))
return int64(*(*uint64)(pa) - *(*uint64)(pb))
}
return int64(*(*uint32)(unsafe.Add(pa, 8)) - *(*uint32)(unsafe.Add(pb, 8)))
} }
func copysha1(dst *byte, src *byte) { func copyhash(dst *byte, src *byte) {
pa, pb := unsafe.Pointer(dst), unsafe.Pointer(src) pa, pb := unsafe.Pointer(dst), unsafe.Pointer(src)
*(*[sha1Size]byte)(pa) = *(*[sha1Size]byte)(pb) *(*[hashSize]byte)(pa) = *(*[hashSize]byte)(pb)
} }
func resetsha1(sha1 *byte) { func resethash(sha1 *byte) {
p := unsafe.Pointer(sha1) p := unsafe.Pointer(sha1)
*(*[sha1Size]byte)(p) = [sha1Size]byte{} *(*[hashSize]byte)(p) = [hashSize]byte{}
} }
// reading table // reading table

7
internal/cache/cache.go vendored
View File

@ -28,15 +28,12 @@ type Cache struct {
db *btree.DB db *btree.DB
} }
// TODO(wdvxdr): cache use md5 key, but btree use sha1 key,
// maybe we can unify to md5 to save some space.
// Insert 添加媒体缓存 // Insert 添加媒体缓存
func (c *Cache) Insert(md5, data []byte) { func (c *Cache) Insert(md5, data []byte) {
c.lock.Lock() c.lock.Lock()
defer c.lock.Unlock() defer c.lock.Unlock()
var hash [20]byte var hash [16]byte
copy(hash[:], md5) copy(hash[:], md5)
c.db.Insert(&hash[0], data) c.db.Insert(&hash[0], data)
} }
@ -46,7 +43,7 @@ func (c *Cache) Get(md5 []byte) []byte {
c.lock.RLock() c.lock.RLock()
defer c.lock.RUnlock() defer c.lock.RUnlock()
var hash [20]byte var hash [16]byte
copy(hash[:], md5) copy(hash[:], md5)
return c.db.Get(&hash[0]) return c.db.Get(&hash[0])
} }