shkong/MiraiGo
1
0
Fork 0
mirror of https://github.com/Mrs4s/MiraiGo.git synced 2025-05-04 11:07:40 +08:00
Code Issues Projects Releases Wiki Activity

zap: reuse some []byte

Browse Source 
优化单线程上传内存使用
This commit is contained in:
wdvxdr 2021-04-15 21:31:44 +08:00
parent 86e0f3327a
commit 2e64a6913d
No known key found for this signature in database
GPG Key ID: 55FF1414A69CEBA6
31 changed files with 360 additions and 248 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
binary/jce/writer.go
View File

@ -243,7 +243,7 @@ func (w *JceWriter) WriteJceStructRaw(s IJceStruct) {
decoderCache.Store(ty2, jceDec) // 存入缓存
}
for _, dec := range jceDec {
var obj = dec.ty.UnsafeIndirect(unsafe.Pointer(uintptr(reflect2.PtrOf(s)) + dec.offset)) // MAGIC!
obj := dec.ty.UnsafeIndirect(unsafe.Pointer(uintptr(reflect2.PtrOf(s)) + dec.offset)) // MAGIC!
if obj != nil {
w.WriteObject(obj, dec.id)
}

97
binary/pool.go Normal file
View File

@ -0,0 +1,97 @@
package binary
import (
"bytes"
"compress/gzip"
"compress/zlib"
"sync"
)
var bufferPool = sync.Pool{
New: func() interface{} {
return new(Writer)
},
}
// NewWriter 从池中取出一个 Writer
func NewWriter() *Writer {
w := bufferPool.Get().(*Writer)
if w == nil {
return new(Writer)
}
return w
}
// PutBuffer 将 Writer 放回池中
func PutBuffer(w *Writer) {
// See https://golang.org/issue/23199
const maxSize = 1 << 16
if w.Cap() < maxSize { // 对于大Buffer直接丢弃
w.Reset()
bufferPool.Put(w)
}
}
type gzipWriter struct {
w *gzip.Writer
buf *bytes.Buffer
}
var gzipPool = sync.Pool{
New: func() interface{} {
buf := new(bytes.Buffer)
w := gzip.NewWriter(buf)
return &gzipWriter{
w: w,
buf: buf,
}
},
}
func acquireGzipWriter() *gzipWriter {
ret := gzipPool.Get().(*gzipWriter)
ret.buf.Reset()
ret.w.Reset(ret.buf)
return ret
}
func releaseGzipWriter(w *gzipWriter) {
// See https://golang.org/issue/23199
const maxSize = 1 << 16
if w.buf.Cap() < maxSize {
w.buf.Reset()
gzipPool.Put(w)
}
}
type zlibWriter struct {
w *zlib.Writer
buf *bytes.Buffer
}
var zlibPool = sync.Pool{
New: func() interface{} {
buf := new(bytes.Buffer)
w := zlib.NewWriter(buf)
return &zlibWriter{
w: w,
buf: buf,
}
},
}
func acquireZlibWriter() *zlibWriter {
ret := zlibPool.Get().(*zlibWriter)
ret.buf.Reset()
ret.w.Reset(ret.buf)
return ret
}
func releaseZlibWriter(w *zlibWriter) {
// See https://golang.org/issue/23199
const maxSize = 1 << 16
if w.buf.Cap() < maxSize {
w.buf.Reset()
zlibPool.Put(w)
}
}

8
binary/tea.go
View File

@ -13,12 +13,9 @@ func xorQ(a, b []byte, c []byte) { // MAGIC
*(*uint64)(unsafe.Pointer((*reflect.SliceHeader)(unsafe.Pointer(&b)).Data))
}
func isZero(a []byte) bool { // MAGIC
return *(*uint64)(unsafe.Pointer((*reflect.SliceHeader)(unsafe.Pointer(&a)).Data)) == 0
}
type TEA [4]uint32
// Encrypt tea 加密
// http://bbs.chinaunix.net/thread-583468-1-1.html
// 感谢xichen大佬对TEA的解释
func (t *TEA) Encrypt(src []byte) (dst []byte) {
@ -86,9 +83,6 @@ func (t *TEA) Decrypt(data []byte) []byte {
xorQ(dst[in:in+8], data[in-8:in], dst[in:in+8])
xorQ(dst[in:in+8], data[in-8:in], tmp)
}
//if !isZero(dst[len(data)-7:]) {
// return nil
//}
return dst[dst[0]&7+3 : len(data)-7]
}

2
binary/tea_test.go
View File

@ -27,7 +27,7 @@ var sampleData = func() [][3]string {
{"LXY1226 Mrs4s", "LXY Testing~", "ab20caa63f3a6503a84f3cb28f9e26b6c18c051e995d1721"},
}
for i, _ := range out {
for i := range out {
c, _ := hex.DecodeString(out[i][ENC])
out[i][ENC] = utils.B2S(c)
}

45
binary/utils.go
View File

@ -8,43 +8,10 @@ import (
"encoding/hex"
"io"
"net"
"sync"
"github.com/Mrs4s/MiraiGo/utils"
)
type gzipWriter struct {
w *gzip.Writer
buf *bytes.Buffer
}
var gzipPool = sync.Pool{
New: func() interface{} {
buf := new(bytes.Buffer)
w := gzip.NewWriter(buf)
return &gzipWriter{
w: w,
buf: buf,
}
},
}
func acquireGzipWriter() *gzipWriter {
ret := gzipPool.Get().(*gzipWriter)
ret.buf.Reset()
ret.w.Reset(ret.buf)
return ret
}
func releaseGzipWriter(w *gzipWriter) {
// See https://golang.org/issue/23199
const maxSize = 1 << 15
if w.buf.Cap() < maxSize {
w.buf.Reset()
gzipPool.Put(w)
}
}
func ZlibUncompress(src []byte) []byte {
b := bytes.NewReader(src)
var out bytes.Buffer
@ -55,11 +22,12 @@ func ZlibUncompress(src []byte) []byte {
}
func ZlibCompress(data []byte) []byte {
buf := new(bytes.Buffer)
w := zlib.NewWriter(buf)
_, _ = w.Write(data)
w.Close()
return buf.Bytes()
zw := acquireZlibWriter()
_, _ = zw.w.Write(data)
_ = zw.w.Close()
ret := append([]byte(nil), zw.buf.Bytes()...)
releaseZlibWriter(zw)
return ret
}
func GZipCompress(data []byte) []byte {
@ -87,7 +55,6 @@ func CalculateImageResourceId(md5 []byte) string {
id = id[:37]
id = append(id, "}.png"...)
return utils.B2S(bytes.ToUpper(id))
}
func GenUUID(uuid []byte) []byte {

44
binary/writer.go
View File

@ -1,33 +1,13 @@
package binary
import (
"bytes"
"encoding/binary"
"encoding/hex"
"sync"
)
type Writer struct {
b []byte
}
var bufferPool = sync.Pool{
New: func() interface{} {
return new(Writer)
},
}
func NewWriter() *Writer {
return bufferPool.Get().(*Writer)
}
func PutBuffer(w *Writer) {
// See https://golang.org/issue/23199
const maxSize = 1 << 16
if cap(w.b) < maxSize { // 对于大Buffer直接丢弃
w.b = w.b[:0]
bufferPool.Put(w)
}
}
// Writer 写入
type Writer bytes.Buffer
func NewWriterF(f func(writer *Writer)) []byte {
w := NewWriter()
@ -38,7 +18,7 @@ func NewWriterF(f func(writer *Writer)) []byte {
}
func (w *Writer) Write(b []byte) {
w.b = append(w.b, b...)
(*bytes.Buffer)(w).Write(b)
}
func (w *Writer) WriteHex(h string) {
@ -47,7 +27,7 @@ func (w *Writer) WriteHex(h string) {
}
func (w *Writer) WriteByte(b byte) {
w.b = append(w.b, b)
(*bytes.Buffer)(w).WriteByte(b)
}
func (w *Writer) WriteUInt16(v uint16) {
@ -129,5 +109,17 @@ func (w *Writer) WriteTlvLimitedSize(data []byte, limit int) {
}
func (w *Writer) Bytes() []byte {
return w.b
return (*bytes.Buffer)(w).Bytes()
}
func (w *Writer) Cap() int {
return (*bytes.Buffer)(w).Cap()
}
func (w *Writer) Reset() {
(*bytes.Buffer)(w).Reset()
}
func (w *Writer) Grow(n int) {
(*bytes.Buffer)(w).Grow(n)
}

7
client/client.go
View File

@ -15,6 +15,9 @@ import (
"sync/atomic"
"time"
jsoniter "github.com/json-iterator/go"
"github.com/pkg/errors"
"github.com/Mrs4s/MiraiGo/binary"
"github.com/Mrs4s/MiraiGo/binary/jce"
"github.com/Mrs4s/MiraiGo/client/pb/msg"
@ -22,8 +25,6 @@ import (
"github.com/Mrs4s/MiraiGo/protocol/crypto"
"github.com/Mrs4s/MiraiGo/protocol/packets"
"github.com/Mrs4s/MiraiGo/utils"
jsoniter "github.com/json-iterator/go"
"github.com/pkg/errors"
)
var json = jsoniter.ConfigFastest
@ -536,7 +537,7 @@ func (c *QQClient) ReloadFriendList() error {
// GetFriendList request friend list
func (c *QQClient) GetFriendList() (*FriendListResponse, error) {
var curFriendCount = 0
curFriendCount := 0
r := &FriendListResponse{}
for {
rsp, err := c.sendAndWait(c.buildFriendGroupListRequestPacket(int16(curFriendCount), 150, 0, 0))

8
client/decoders.go
View File

@ -447,7 +447,7 @@ func decodeFriendGroupListResponse(_ *QQClient, _ *incomingPacketInfo, payload [
totalFriendCount := r.ReadInt16(5)
friends := []jce.FriendInfo{}
r.ReadSlice(&friends, 7)
var l = make([]*FriendInfo, 0, len(friends))
l := make([]*FriendInfo, 0, len(friends))
for _, f := range friends {
l = append(l, &FriendInfo{
Uin: f.FriendUin,
@ -474,7 +474,7 @@ func decodeGroupListResponse(c *QQClient, _ *incomingPacketInfo, payload []byte)
groups := []jce.TroopNumber{}
r.ReadSlice(&vecCookie, 4)
r.ReadSlice(&groups, 5)
var l = make([]*GroupInfo, 0, len(groups))
l := make([]*GroupInfo, 0, len(groups))
for _, g := range groups {
l = append(l, &GroupInfo{
Uin: g.GroupUin,
@ -507,7 +507,7 @@ func decodeGroupMemberListResponse(_ *QQClient, _ *incomingPacketInfo, payload [
members := []jce.TroopMemberInfo{}
r.ReadSlice(&members, 3)
next := r.ReadInt64(4)
var l = make([]*GroupMemberInfo, 0, len(members))
l := make([]*GroupMemberInfo, 0, len(members))
for _, m := range members {
l = append(l, &GroupMemberInfo{
Uin: m.MemberUin,
@ -677,7 +677,7 @@ func decodeOnlinePushTransPacket(c *QQClient, _ *incomingPacketInfo, payload []b
if info.GetMsgType() == 44 {
data.ReadBytes(5)
var4 := int32(data.ReadByte())
var var5 = int64(0)
var5 := int64(0)
target := int64(uint32(data.ReadInt32()))
if var4 != 0 && var4 != 1 {
var5 = int64(uint32(data.ReadInt32()))

5
client/face.go
View File

@ -3,10 +3,11 @@ package client
import (
"fmt"
"github.com/Mrs4s/MiraiGo/client/pb/faceroam"
"github.com/Mrs4s/MiraiGo/protocol/packets"
"github.com/golang/protobuf/proto"
"github.com/pkg/errors"
"github.com/Mrs4s/MiraiGo/client/pb/faceroam"
"github.com/Mrs4s/MiraiGo/protocol/packets"
)
type CustomFace struct {

10
client/global.go
View File

@ -156,8 +156,10 @@ var SystemDeviceInfo = &DeviceInfo{
},
}
var EmptyBytes = []byte{}
var NumberRange = "0123456789"
var (
EmptyBytes = []byte{}
NumberRange = "0123456789"
)
func init() {
r := make([]byte, 16)
@ -404,7 +406,7 @@ func GenIMEI() string {
sum += toAdd
final.WriteString(fmt.Sprintf("%d", toAdd)) // and even printing them here!
}
var ctrlDigit = (sum * 9) % 10 // calculating the control digit
ctrlDigit := (sum * 9) % 10 // calculating the control digit
final.WriteString(fmt.Sprintf("%d", ctrlDigit))
return final.String()
}
@ -443,7 +445,7 @@ func getSSOAddress() ([]*net.TCPAddr, error) {
reader := jce.NewJceReader(data.Map["HttpServerListRes"][1:])
servers := []jce.SsoServerInfo{}
reader.ReadSlice(&servers, 2)
var adds = make([]*net.TCPAddr, 0, len(servers))
adds := make([]*net.TCPAddr, 0, len(servers))
for _, s := range servers {
if strings.Contains(s.Server, "com") {
continue

5
client/group_file.go
View File

@ -12,11 +12,12 @@ import (
"google.golang.org/protobuf/proto"
"github.com/pkg/errors"
"github.com/Mrs4s/MiraiGo/client/pb/exciting"
"github.com/Mrs4s/MiraiGo/client/pb/oidb"
"github.com/Mrs4s/MiraiGo/protocol/packets"
"github.com/Mrs4s/MiraiGo/utils"
"github.com/pkg/errors"
)
type (
@ -173,7 +174,7 @@ func (fs *GroupFileSystem) UploadFile(p, name, folderId string) error {
defer fileSingleFlight.Delete(p)
}
file, err := os.OpenFile(p, os.O_RDONLY, 0666)
file, err := os.OpenFile(p, os.O_RDONLY, 0o666)
if err != nil {
return errors.Wrap(err, "open file error")
}

2
client/group_msg.go
View File

@ -631,7 +631,7 @@ func (c *QQClient) GetGroupEssenceMsgList(groupCode int64) ([]GroupDigest, error
}
rsp = rsp[bytes.Index(rsp, []byte("window.__INITIAL_STATE__={"))+25:]
rsp = rsp[:bytes.Index(rsp, []byte("</script>"))]
var data = &struct {
data := &struct {
List []GroupDigest `json:"msgList"`
}{}
err = json.Unmarshal(rsp, data)

5
client/group_notice.go
View File

@ -11,8 +11,9 @@ import (
"net/url"
"strconv"
"github.com/Mrs4s/MiraiGo/utils"
"github.com/pkg/errors"
"github.com/Mrs4s/MiraiGo/utils"
)
type noticePicUpResponse struct {
@ -29,7 +30,7 @@ type noticeImage struct {
func (c *QQClient) uploadGroupNoticePic(img []byte) (*noticeImage, error) {
buf := new(bytes.Buffer)
var w = multipart.NewWriter(buf)
w := multipart.NewWriter(buf)
err := w.WriteField("bkn", strconv.Itoa(c.getCSRFToken()))
if err != nil {
return nil, errors.Wrap(err, "write multipart<bkn> failed")

68
client/highway.go
View File

@ -37,7 +37,7 @@ func (c *QQClient) highwayUploadStream(ip uint32, port int, updKey []byte, strea
h := md5.New()
length, _ := io.Copy(h, stream)
fh := h.Sum(nil)
chunkSize := 8192 * 8
const chunkSize = 8192 * 8
_, _ = stream.Seek(0, io.SeekStart)
conn, err := net.DialTCP("tcp", nil, &addr)
if err != nil {
@ -46,8 +46,11 @@ func (c *QQClient) highwayUploadStream(ip uint32, port int, updKey []byte, strea
defer conn.Close()
offset := 0
reader := binary.NewNetworkReader(conn)
for {
chunk := make([]byte, chunkSize)
w := binary.NewWriter()
defer binary.PutBuffer(w)
for {
chunk = chunk[:chunkSize]
rl, err := io.ReadFull(stream, chunk)
if errors.Is(err, io.EOF) {
break
@ -78,14 +81,14 @@ func (c *QQClient) highwayUploadStream(ip uint32, port int, updKey []byte, strea
ReqExtendinfo: EmptyBytes,
})
offset += rl
_, err = conn.Write(binary.NewWriterF(func(w *binary.Writer) {
w.Reset()
w.WriteByte(40)
w.WriteUInt32(uint32(len(head)))
w.WriteUInt32(uint32(len(chunk)))
w.Write(head)
w.Write(chunk)
w.WriteByte(41)
}))
_, err = conn.Write(w.Bytes())
if err != nil {
return errors.Wrap(err, "write conn error")
}
@ -100,7 +103,7 @@ func (c *QQClient) highwayUploadStream(ip uint32, port int, updKey []byte, strea
return nil
}
func (c *QQClient) highwayUploadByBDH(stream io.ReadSeeker, cmdId int32, ticket, ext []byte, encrypt bool) ([]byte, error) {
func (c *QQClient) highwayUploadByBDH(stream io.Reader, length int64, cmdId int32, ticket, sum, ext []byte, encrypt bool) ([]byte, error) {
if len(c.srvSsoAddrs) == 0 {
return nil, errors.New("srv addrs not found. maybe miss some packet?")
}
@ -110,11 +113,7 @@ func (c *QQClient) highwayUploadByBDH(stream io.ReadSeeker, cmdId int32, ticket,
}
ext = binary.NewTeaCipher(c.highwaySession.SessionKey).Encrypt(ext)
}
h := md5.New()
length, _ := io.Copy(h, stream)
fh := h.Sum(nil)
chunkSize := 8192 * 16
_, _ = stream.Seek(0, io.SeekStart)
const chunkSize = 8192 * 16
conn, err := net.DialTimeout("tcp", c.srvSsoAddrs[0], time.Second*20)
if err != nil {
return nil, errors.Wrap(err, "connect error")
@ -129,8 +128,11 @@ func (c *QQClient) highwayUploadByBDH(stream io.ReadSeeker, cmdId int32, ticket,
return nil, errors.Wrap(err, "echo error")
}
var rspExt []byte
for {
chunk := make([]byte, chunkSize)
w := binary.NewWriter()
defer binary.PutBuffer(w)
for {
chunk = chunk[:chunkSize]
rl, err := io.ReadFull(stream, chunk)
if errors.Is(err, io.EOF) {
break
@ -156,19 +158,19 @@ func (c *QQClient) highwayUploadByBDH(stream io.ReadSeeker, cmdId int32, ticket,
Datalength: int32(rl),
Serviceticket: ticket,
Md5: ch[:],
FileMd5: fh[:],
FileMd5: sum,
},
ReqExtendinfo: ext,
})
offset += rl
_, err = conn.Write(binary.NewWriterF(func(w *binary.Writer) {
w.Reset()
w.WriteByte(40)
w.WriteUInt32(uint32(len(head)))
w.WriteUInt32(uint32(len(chunk)))
w.Write(head)
w.Write(chunk)
w.WriteByte(41)
}))
_, err = conn.Write(w.Bytes())
if err != nil {
return nil, errors.Wrap(err, "write conn error")
}
@ -203,22 +205,23 @@ func (c *QQClient) highwayUploadFileMultiThreadingByBDH(path string, cmdId int32
if err != nil {
return nil, errors.Wrap(err, "get stat error")
}
file, err := os.OpenFile(path, os.O_RDONLY, 0666)
file, err := os.OpenFile(path, os.O_RDONLY, 0o666)
if err != nil {
return nil, errors.Wrap(err, "open file error")
}
defer file.Close()
h := md5.New()
length, _ := io.Copy(h, file)
fh := h.Sum(nil)
_, _ = file.Seek(0, io.SeekStart)
if stat.Size() < 1024*1024*3 {
return c.highwayUploadByBDH(file, cmdId, ticket, ext, false)
return c.highwayUploadByBDH(file, length, cmdId, ticket, fh, ext, false)
}
type BlockMetaData struct {
Id int
BeginOffset int64
EndOffset int64
}
h := md5.New()
_, _ = io.Copy(h, file)
fh := h.Sum(nil)
const blockSize int64 = 1024 * 512
var (
blocks []*BlockMetaData
@ -251,7 +254,7 @@ func (c *QQClient) highwayUploadFileMultiThreadingByBDH(path string, cmdId int32
return errors.Wrap(err, "connect error")
}
defer conn.Close()
chunk, _ := os.OpenFile(path, os.O_RDONLY, 0666)
chunk, _ := os.OpenFile(path, os.O_RDONLY, 0o666)
defer chunk.Close()
reader := binary.NewNetworkReader(conn)
if err = c.highwaySendHeartbreak(conn); err != nil {
@ -278,7 +281,7 @@ func (c *QQClient) highwayUploadFileMultiThreadingByBDH(path string, cmdId int32
}
buffer := make([]byte, blockSize)
_, _ = chunk.Seek(block.BeginOffset, io.SeekStart)
ri, err := io.ReadFull(chunk, buffer)
ri, err := io.ReadFull(chunk, buffer) // todo: reuse buffer
if err != nil {
if err == io.EOF {
break
@ -311,14 +314,15 @@ func (c *QQClient) highwayUploadFileMultiThreadingByBDH(path string, cmdId int32
},
ReqExtendinfo: ext,
})
_, err = conn.Write(binary.NewWriterF(func(w *binary.Writer) {
w := binary.NewWriter()
w.WriteByte(40)
w.WriteUInt32(uint32(len(head)))
w.WriteUInt32(uint32(len(buffer)))
w.Write(head)
w.Write(buffer)
w.WriteByte(41)
}))
_, err = conn.Write(w.Bytes())
binary.PutBuffer(w)
if err != nil {
return errors.Wrap(err, "write conn error")
}
@ -364,13 +368,14 @@ func (c *QQClient) highwaySendHeartbreak(conn net.Conn) error {
LocaleId: 2052,
},
})
_, err := conn.Write(binary.NewWriterF(func(w *binary.Writer) {
w := binary.NewWriter()
w.WriteByte(40)
w.WriteUInt32(uint32(len(head)))
w.WriteUInt32(0)
w.Write(head)
w.WriteByte(41)
}))
_, err := conn.Write(w.Bytes())
binary.PutBuffer(w)
return err
}
@ -400,8 +405,12 @@ func (c *QQClient) excitingUploadStream(stream io.ReadSeeker, cmdId int32, ticke
offset int64 = 0
chunkSize = 524288
)
for {
chunk := make([]byte, chunkSize)
w := binary.NewWriter()
w.Reset()
w.Grow(600 * 1024) // 复用,600k 不要放回池中
for {
chunk = chunk[:chunkSize]
rl, err := io.ReadFull(stream, chunk)
if err == io.EOF {
break
@ -432,15 +441,14 @@ func (c *QQClient) excitingUploadStream(stream io.ReadSeeker, cmdId int32, ticke
ReqExtendinfo: ext,
})
offset += int64(rl)
io.Pipe()
req, _ := http.NewRequest("POST", url, bytes.NewReader(binary.NewWriterF(func(w *binary.Writer) {
w.Reset()
w.WriteByte(40)
w.WriteUInt32(uint32(len(head)))
w.WriteUInt32(uint32(len(chunk)))
w.Write(head)
w.Write(chunk)
w.WriteByte(41)
})))
req, _ := http.NewRequest("POST", url, bytes.NewReader(w.Bytes()))
req.Header.Set("Accept", "*/*")
req.Header.Set("Connection", "Keep-Alive")
req.Header.Set("User-Agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)")
@ -504,7 +512,7 @@ func (c *QQClient) uploadGroupHeadPortrait(groupCode int64, img []byte) error {
groupCode,
len(img),
)
req, err := http.NewRequest("POST", url, bytes.NewReader(img))
req, _ := http.NewRequest("POST", url, bytes.NewReader(img))
req.Header["User-Agent"] = []string{"Dalvik/2.1.0 (Linux; U; Android 7.1.2; PCRT00 Build/N2G48H)"}
req.Header["Content-Type"] = []string{"multipart/form-data;boundary=****"}
rsp, err := http.DefaultClient.Do(req)

27
client/image.go
View File

@ -13,13 +13,14 @@ import (
"github.com/Mrs4s/MiraiGo/client/pb/highway"
"github.com/Mrs4s/MiraiGo/client/pb/oidb"
"github.com/pkg/errors"
"google.golang.org/protobuf/proto"
"github.com/Mrs4s/MiraiGo/binary"
"github.com/Mrs4s/MiraiGo/client/pb"
"github.com/Mrs4s/MiraiGo/message"
"github.com/Mrs4s/MiraiGo/protocol/packets"
"github.com/Mrs4s/MiraiGo/utils"
"github.com/pkg/errors"
"google.golang.org/protobuf/proto"
)
func init() {
@ -48,7 +49,7 @@ func (c *QQClient) UploadGroupImage(groupCode int64, img io.ReadSeeker) (*messag
c.srvSsoAddrs = append(c.srvSsoAddrs, fmt.Sprintf("%v:%v", binary.UInt32ToIPV4Address(uint32(addr)), rsp.UploadPort[i]))
}
}
if _, err = c.highwayUploadByBDH(img, 2, rsp.UploadKey, EmptyBytes, false); err == nil {
if _, err = c.highwayUploadByBDH(img, length, 2, rsp.UploadKey, EmptyBytes, fh, false); err == nil {
goto ok
}
return nil, errors.New("upload failed")
@ -66,7 +67,7 @@ ok:
}
func (c *QQClient) UploadGroupImageByFile(groupCode int64, path string) (*message.GroupImageElement, error) {
img, err := os.OpenFile(path, os.O_RDONLY, 0666)
img, err := os.OpenFile(path, os.O_RDONLY, 0o666)
if err != nil {
return nil, err
}
@ -137,10 +138,11 @@ func (c *QQClient) ImageOcr(img interface{}) (*OcrResponse, error) {
switch e := img.(type) {
case *message.GroupImageElement:
url = e.Url
if b, err := utils.HttpGetBytes(e.Url, ""); err == nil {
if url, err = c.uploadOcrImage(bytes.NewReader(b)); err != nil {
if b, err := utils.HTTPGetReadCloser(e.Url, ""); err == nil {
if url, err = c.uploadOcrImage(b, int64(e.Size), e.Md5); err != nil {
url = e.Url
}
_ = b.Close()
}
rsp, err := c.sendAndWait(c.buildImageOcrRequestPacket(url, strings.ToUpper(hex.EncodeToString(e.Md5)), e.Size, e.Width, e.Height))
if err != nil {
@ -149,10 +151,11 @@ func (c *QQClient) ImageOcr(img interface{}) (*OcrResponse, error) {
return rsp.(*OcrResponse), nil
case *message.ImageElement:
url = e.Url
if b, err := utils.HttpGetBytes(e.Url, ""); err == nil {
if url, err = c.uploadOcrImage(bytes.NewReader(b)); err != nil {
if b, err := utils.HTTPGetReadCloser(e.Url, ""); err == nil {
if url, err = c.uploadOcrImage(b, int64(e.Size), e.Md5); err != nil {
url = e.Url
}
_ = b.Close()
}
rsp, err := c.sendAndWait(c.buildImageOcrRequestPacket(url, strings.ToUpper(hex.EncodeToString(e.Md5)), e.Size, e.Width, e.Height))
if err != nil {
@ -232,14 +235,14 @@ func (c *QQClient) buildGroupImageStorePacket(groupCode int64, md5 []byte, size
return seq, packet
}
func (c *QQClient) uploadOcrImage(img io.ReadSeeker) (string, error) {
func (c *QQClient) uploadOcrImage(img io.Reader, length int64, sum []byte) (string, error) {
r := make([]byte, 16)
rand.Read(r)
ext, _ := proto.Marshal(&highway.CommFileExtReq{
ActionType: proto.Uint32(0),
Uuid: binary.GenUUID(r),
})
rsp, err := c.highwayUploadByBDH(img, 76, c.highwaySession.SigSession, ext, false)
rsp, err := c.highwayUploadByBDH(img, length, 76, c.highwaySession.SigSession, sum, ext, false)
if err != nil {
return "", errors.Wrap(err, "upload ocr image error")
}
@ -316,9 +319,9 @@ func decodeImageOcrResponse(_ *QQClient, _ *incomingPacketInfo, payload []byte)
if rsp.RetCode != 0 {
return nil, errors.Errorf("server error, code: %v msg: %v", rsp.RetCode, rsp.ErrMsg)
}
var texts = make([]*TextDetection, 0, len(rsp.OcrRspBody.TextDetections))
texts := make([]*TextDetection, 0, len(rsp.OcrRspBody.TextDetections))
for _, text := range rsp.OcrRspBody.TextDetections {
var points = make([]*Coordinate, 0, len(text.Polygon.Coordinates))
points := make([]*Coordinate, 0, len(text.Polygon.Coordinates))
for _, c := range text.Polygon.Coordinates {
points = append(points, &Coordinate{
X: c.X,

2
client/notify.go
View File

@ -40,7 +40,7 @@ type (
// grayTipProcessor 提取出来专门用于处理群内 notify tips
func (c *QQClient) grayTipProcessor(groupID int64, tipInfo *notify.GeneralGrayTipInfo) {
if tipInfo.BusiType == 12 && tipInfo.BusiId == 1061 {
var sender = int64(0)
sender := int64(0)
receiver := c.Uin
for _, templ := range tipInfo.MsgTemplParam {
if templ.Name == "uin_str1" {

7
client/online_push.go
View File

@ -7,12 +7,13 @@ import (
"github.com/Mrs4s/MiraiGo/client/pb/msgtype0x210"
"github.com/pkg/errors"
"google.golang.org/protobuf/proto"
"github.com/Mrs4s/MiraiGo/binary"
"github.com/Mrs4s/MiraiGo/binary/jce"
"github.com/Mrs4s/MiraiGo/client/pb"
"github.com/Mrs4s/MiraiGo/client/pb/notify"
"github.com/pkg/errors"
"google.golang.org/protobuf/proto"
)
var msg0x210Decoders = map[int64]func(*QQClient, []byte) error{
@ -90,7 +91,7 @@ func decodeOnlinePushReqPacket(c *QQClient, info *incomingPacketInfo, payload []
}
}
if b.QqGroupDigestMsg != nil {
var digest = b.QqGroupDigestMsg
digest := b.QqGroupDigestMsg
c.dispatchGroupDigestEvent(&GroupDigestEvent{
GroupCode: int64(digest.GroupCode),
MessageID: int32(digest.Seq),

3
client/private_msg.go
View File

@ -6,10 +6,11 @@ import (
"github.com/pkg/errors"
"github.com/golang/protobuf/proto"
"github.com/Mrs4s/MiraiGo/client/pb/msg"
"github.com/Mrs4s/MiraiGo/message"
"github.com/Mrs4s/MiraiGo/protocol/packets"
"github.com/golang/protobuf/proto"
)
func (c *QQClient) SendPrivateMessage(target int64, m *message.SendingMessage) *message.PrivateMessage {

23
client/ptt.go
View File

@ -31,7 +31,7 @@ func (c *QQClient) UploadGroupPtt(groupCode int64, voice io.ReadSeeker) (*messag
fh := h.Sum(nil)
_, _ = voice.Seek(0, io.SeekStart)
ext := c.buildGroupPttStoreBDHExt(groupCode, fh[:], int32(length), 0, int32(length))
rsp, err := c.highwayUploadByBDH(voice, 29, c.highwaySession.SigSession, ext, false)
rsp, err := c.highwayUploadByBDH(voice, length, 29, c.highwaySession.SigSession, fh, ext, false)
if err != nil {
return nil, err
}
@ -55,7 +55,8 @@ func (c *QQClient) UploadGroupPtt(groupCode int64, voice io.ReadSeeker) (*messag
GroupFileKey: pkt.MsgTryUpPttRsp[0].FileKey,
BoolValid: proto.Bool(true),
PbReserve: []byte{8, 0, 40, 0, 56, 0},
}}, nil
},
}, nil
}
// UploadPrivatePtt 将语音数据使用好友语音通道上传到服务器, 返回 message.PrivateVoiceElement 可直接发送
@ -65,7 +66,7 @@ func (c *QQClient) UploadPrivatePtt(target int64, voice io.ReadSeeker) (*message
fh := h.Sum(nil)
_, _ = voice.Seek(0, io.SeekStart)
ext := c.buildC2CPttStoreBDHExt(target, fh[:], int32(length), int32(length))
rsp, err := c.highwayUploadByBDH(voice, 26, c.highwaySession.SigSession, ext, false)
rsp, err := c.highwayUploadByBDH(voice, length, 26, c.highwaySession.SigSession, fh, ext, false)
if err != nil {
return nil, err
}
@ -89,7 +90,8 @@ func (c *QQClient) UploadPrivatePtt(target int64, voice io.ReadSeeker) (*message
FileSize: proto.Int32(int32(length)),
// Reserve: constructPTTExtraInfo(1, int32(len(voice))), // todo length
BoolValid: proto.Bool(true),
}}, nil
},
}, nil
}
// UploadGroupShortVideo 将视频和封面上传到服务器, 返回 message.ShortVideoElement 可直接发送
@ -117,22 +119,27 @@ func (c *QQClient) UploadGroupShortVideo(groupCode int64, video, thumb io.ReadSe
}
ext, _ := proto.Marshal(c.buildPttGroupShortVideoProto(videoHash, thumbHash, groupCode, videoLen, thumbLen).PttShortVideoUploadReq)
var hwRsp []byte
multi := utils.MultiReadSeeker(thumb, video)
h := md5.New()
length, _ := io.Copy(h, multi)
fh := h.Sum(nil)
_, _ = multi.Seek(0, io.SeekStart)
if cache != "" {
var file *os.File
file, err = os.OpenFile(cache, os.O_WRONLY|os.O_CREATE, 0666)
file, err = os.OpenFile(cache, os.O_WRONLY|os.O_CREATE, 0o666)
cp := func() error {
_, err := io.Copy(file, utils.MultiReadSeeker(thumb, video))
return err
}
if err != nil || cp() != nil {
hwRsp, err = c.highwayUploadByBDH(utils.MultiReadSeeker(thumb, video), 25, c.highwaySession.SigSession, ext, true)
hwRsp, err = c.highwayUploadByBDH(multi, length, 25, c.highwaySession.SigSession, fh, ext, true)
} else {
_ = file.Close()
hwRsp, err = c.highwayUploadFileMultiThreadingByBDH(cache, 25, 8, c.highwaySession.SigSession, ext, true)
_ = os.Remove(cache)
}
} else {
hwRsp, err = c.highwayUploadByBDH(utils.MultiReadSeeker(thumb, video), 25, c.highwaySession.SigSession, ext, true)
hwRsp, err = c.highwayUploadByBDH(multi, length, 25, c.highwaySession.SigSession, fh, ext, true)
}
if err != nil {
return nil, errors.Wrap(err, "upload video file error")
@ -279,7 +286,7 @@ func decodeGroupPttStoreResponse(_ *QQClient, _ *incomingPacketInfo, payload []b
if rsp.BoolFileExit {
return pttUploadResponse{IsExists: true}, nil
}
var ip = make([]string, 0, len(rsp.Uint32UpIp))
ip := make([]string, 0, len(rsp.Uint32UpIp))
for _, i := range rsp.Uint32UpIp {
ip = append(ip, binary.UInt32ToIPV4Address(uint32(i)))
}

5
client/sync.go
View File

@ -56,7 +56,7 @@ func (c *QQClient) GetAllowedClients() ([]*OtherClientInfo, error) {
return nil, err
}
list := i.([]jce.SvcDevLoginInfo)
var ret = make([]*OtherClientInfo, 0, len(list))
ret := make([]*OtherClientInfo, 0, len(list))
for _, l := range list {
ret = append(ret, &OtherClientInfo{
AppId: l.AppId,
@ -208,7 +208,8 @@ func (c *QQClient) buildSyncMsgRequestPacket() (uint16, []byte) {
Type: proto.Uint32(283),
Version: proto.Uint32(0),
},
}})
},
})
regReq := &jce.SvcReqRegisterNew{
RequestOptional: 128 | 64 | 256 | 2 | 8192 | 16384 | 65536,
DisGroupMsgFilter: 1,

1
message/elements.go
View File

@ -271,6 +271,7 @@ func NewUrlShare(url, title, content, image string) *ServiceElement {
SubType: "UrlShare",
}
}
func NewRichXml(template string, ResId int64) *ServiceElement {
if ResId == 0 {
ResId = 60 // 默认值60

2
message/generate.go
View File

@ -36,7 +36,7 @@ var faceMap = map[int]string{
`
func main() {
f, _ := os.OpenFile("face.go", os.O_WRONLY|os.O_CREATE|os.O_SYNC|os.O_TRUNC, 0755)
f, _ := os.OpenFile("face.go", os.O_WRONLY|os.O_CREATE|os.O_SYNC|os.O_TRUNC, 0o755)
defer func() { _ = f.Close() }()
resp, err := http.Get(faceDownloadUrl)
if err != nil {

2
message/message.go
View File

@ -590,7 +590,7 @@ func (forMsg *ForwardMessage) CalculateValidationDataForward(seq, random int32,
}
func (forMsg *ForwardMessage) packForwardMsg(seq int32, random int32, groupCode int64) []*msg.Message {
var msgs = make([]*msg.Message, 0, len(forMsg.Nodes))
msgs := make([]*msg.Message, 0, len(forMsg.Nodes))
for _, node := range forMsg.Nodes {
msgs = append(msgs, &msg.Message{
Head: &msg.MessageHead{

6
message/pack.go
View File

@ -8,9 +8,11 @@ import (
"google.golang.org/protobuf/proto"
)
var imgOld = []byte{0x15, 0x36, 0x20, 0x39, 0x32, 0x6B, 0x41, 0x31, 0x00, 0x38, 0x37, 0x32, 0x66, 0x30, 0x36, 0x36, 0x30, 0x33, 0x61, 0x65, 0x31, 0x30, 0x33, 0x62, 0x37, 0x20, 0x20, 0x20, 0x20, 0x20,
var imgOld = []byte{
0x15, 0x36, 0x20, 0x39, 0x32, 0x6B, 0x41, 0x31, 0x00, 0x38, 0x37, 0x32, 0x66, 0x30, 0x36, 0x36, 0x30, 0x33, 0x61, 0x65, 0x31, 0x30, 0x33, 0x62, 0x37, 0x20, 0x20, 0x20, 0x20, 0x20,
0x20, 0x35, 0x30, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x7B, 0x30, 0x31, 0x45, 0x39, 0x34, 0x35, 0x31, 0x42, 0x2D, 0x37, 0x30, 0x45, 0x44,
0x2D, 0x45, 0x41, 0x45, 0x33, 0x2D, 0x42, 0x33, 0x37, 0x43, 0x2D, 0x31, 0x30, 0x31, 0x46, 0x31, 0x45, 0x45, 0x42, 0x46, 0x35, 0x42, 0x35, 0x7D, 0x2E, 0x70, 0x6E, 0x67, 0x41}
0x2D, 0x45, 0x41, 0x45, 0x33, 0x2D, 0x42, 0x33, 0x37, 0x43, 0x2D, 0x31, 0x30, 0x31, 0x46, 0x31, 0x45, 0x45, 0x42, 0x46, 0x35, 0x42, 0x35, 0x7D, 0x2E, 0x70, 0x6E, 0x67, 0x41,
}
func (e *TextElement) Pack() (r []*msg.Elem) {
r = append(r, &msg.Elem{

2
protocol/crypto/crypto.go
View File

@ -96,7 +96,7 @@ func (e *EncryptECDH) FetchPubKey(uin int64) {
return
}
defer resp.Body.Close()
var pubKey = pubKeyResp{}
pubKey := pubKeyResp{}
err = jsoniter.NewDecoder(resp.Body).Decode(&pubKey)
if err != nil {
return

12
protocol/packets/global.go
View File

@ -8,11 +8,13 @@ import (
"github.com/pkg/errors"
)
var ErrUnknownFlag = errors.New("unknown flag")
var ErrInvalidPayload = errors.New("invalid payload")
var ErrDecryptFailed = errors.New("decrypt failed")
var ErrSessionExpired = errors.New("session expired")
var ErrPacketDropped = errors.New("packet dropped")
var (
ErrUnknownFlag = errors.New("unknown flag")
ErrInvalidPayload = errors.New("invalid payload")
ErrDecryptFailed = errors.New("decrypt failed")
ErrSessionExpired = errors.New("session expired")
ErrPacketDropped = errors.New("packet dropped")
)
type ISendingPacket interface {
CommandId() uint16

76
utils/http.go
View File

@ -3,6 +3,7 @@ package utils
import (
"bytes"
"compress/gzip"
"io"
"io/ioutil"
"net/http"
"strings"
@ -19,32 +20,12 @@ var client = &http.Client{
// HttpGetBytes 带 cookie 的 GET 请求
func HttpGetBytes(url, cookie string) ([]byte, error) {
req, err := http.NewRequest("GET", url, nil)
body, err := HTTPGetReadCloser(url, cookie)
defer func() { _ = body.Close() }()
if err != nil {
return nil, err
}
req.Header["User-Agent"] = []string{"QQ/8.2.0.1296 CFNetwork/1126"}
req.Header["Net-Type"] = []string{"Wifi"}
if cookie != "" {
req.Header["Cookie"] = []string{cookie}
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if strings.Contains(resp.Header.Get("Content-Encoding"), "gzip") {
buffer := bytes.NewBuffer(body)
r, _ := gzip.NewReader(buffer)
defer r.Close()
unCom, err := ioutil.ReadAll(r)
return unCom, err
}
return body, nil
return io.ReadAll(body)
}
func HttpPostBytes(url string, data []byte) ([]byte, error) {
@ -105,3 +86,52 @@ func HttpPostBytesWithCookie(url string, data []byte, cookie string, contentType
}
return body, nil
}
type gzipCloser struct {
f io.Closer
r *gzip.Reader
}
// NewGzipReadCloser 从 io.ReadCloser 创建 gunzip io.ReadCloser
func NewGzipReadCloser(reader io.ReadCloser) (io.ReadCloser, error) {
gzipReader, err := gzip.NewReader(reader)
if err != nil {
return nil, err
}
return &gzipCloser{
f: reader,
r: gzipReader,
}, nil
}
// Read impls io.Reader
func (g *gzipCloser) Read(p []byte) (n int, err error) {
return g.r.Read(p)
}
// Close impls io.Closer
func (g *gzipCloser) Close() error {
_ = g.f.Close()
return g.r.Close()
}
// HTTPGetReadCloser 从 Http url 获取 io.ReadCloser
func HTTPGetReadCloser(url string, cookie string) (io.ReadCloser, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header["User-Agent"] = []string{"QQ/8.2.0.1296 CFNetwork/1126"}
req.Header["Net-Type"] = []string{"Wifi"}
if cookie != "" {
req.Header["Cookie"] = []string{cookie}
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
if strings.Contains(resp.Header.Get("Content-Encoding"), "gzip") {
return NewGzipReadCloser(resp.Body)
}
return resp.Body, err
}