Skip to content

Commit df8f6af

Browse files
fumiamafumiama
committed
optimize: memory consumption
1 parent d65fc4d commit df8f6af

19 files changed

Lines changed: 204 additions & 242 deletions

File tree

‎go.mod‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@ go 1.21
55
require (
66
github.com/FloatTech/ttl v0.0.0-20250224045156-012b1463287d
77
github.com/RomiChan/syncx v0.0.0-20240418144900-b7402ffdebc7
8-
github.com/fumiama/blake2b-simd v0.0.0-20220412110131-4481822068bb
8+
github.com/fumiama/blake2b-simd v0.0.0-20250228045919-a5dcaba5419a
99
github.com/fumiama/go-base16384 v1.7.0
1010
github.com/fumiama/go-x25519 v1.0.0
11-
github.com/fumiama/orbyte v0.0.0-20250414141219-63dd01e81ea2
11+
github.com/fumiama/orbyte v0.0.0-20250512155242-23a2b7120589
1212
github.com/fumiama/water v0.0.0-20211231134027-da391938d6ac
1313
github.com/klauspost/compress v1.17.9
1414
github.com/sirupsen/logrus v1.9.3

‎go.sum‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,14 @@ github.com/RomiChan/syncx v0.0.0-20240418144900-b7402ffdebc7/go.mod h1:vD7Ra3Q9o
55
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
66
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
77
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
8-
github.com/fumiama/blake2b-simd v0.0.0-20220412110131-4481822068bb h1:Gd7gIiR68ErTWtxnl/PCycMog6IRdK1ApxrvoAy0lFY=
9-
github.com/fumiama/blake2b-simd v0.0.0-20220412110131-4481822068bb/go.mod h1:Olmv2uLdFllRsvwhzOvG/O/Nvgzg0ViokUL4+hiaRSE=
8+
github.com/fumiama/blake2b-simd v0.0.0-20250228045919-a5dcaba5419a h1:hhCWoHNGDATjUitm8rKwrf5xRnuxO7P6UHIusLXuqag=
9+
github.com/fumiama/blake2b-simd v0.0.0-20250228045919-a5dcaba5419a/go.mod h1:Olmv2uLdFllRsvwhzOvG/O/Nvgzg0ViokUL4+hiaRSE=
1010
github.com/fumiama/go-base16384 v1.7.0 h1:6fep7XPQWxRlh4Hu+KsdH+6+YdUp+w6CwRXtMWSsXCA=
1111
github.com/fumiama/go-base16384 v1.7.0/go.mod h1:OEn+947GV5gsbTAnyuUW/SrfxJYUdYupSIQXOuGOcXM=
1212
github.com/fumiama/go-x25519 v1.0.0 h1:hiGg9EhseVmGCc8T1jECVkj8Keu/aJ1ZK05RM8Vuavo=
1313
github.com/fumiama/go-x25519 v1.0.0/go.mod h1:8VOhfyGZzw4IUs4nCjQFqW9cA3V/QpSCtP3fo2dLNg4=
14-
github.com/fumiama/orbyte v0.0.0-20250414141219-63dd01e81ea2 h1:A0jvi8f+FHNtPJBKujPYshazTmujSE/trf5xkmrJvSQ=
15-
github.com/fumiama/orbyte v0.0.0-20250414141219-63dd01e81ea2/go.mod h1:FOjdw7KdCbK2eH3gRPhwFNCoXKpu9sN5vPH4El/8e0c=
14+
github.com/fumiama/orbyte v0.0.0-20250512155242-23a2b7120589 h1:ifo33HcA8HFYWaTe4fsJjozTUaOBlA0dFa1BcTZDEHs=
15+
github.com/fumiama/orbyte v0.0.0-20250512155242-23a2b7120589/go.mod h1:FOjdw7KdCbK2eH3gRPhwFNCoXKpu9sN5vPH4El/8e0c=
1616
github.com/fumiama/water v0.0.0-20211231134027-da391938d6ac h1:A/5A0rODsg+EQHH61Ew5mMUtDpRXaSNqHhPvW+fN4C4=
1717
github.com/fumiama/water v0.0.0-20211231134027-da391938d6ac/go.mod h1:BBnNY9PwK+UUn4trAU+H0qsMEypm7+3Bj1bVFuJItlo=
1818
github.com/fumiama/wintun v0.0.0-20211229152851-8bc97c8034c0 h1:WfrSFlIlCAtg6Rt2IGna0HhJYSDE45YVHiYqO4wwsEw=

‎gold/head/box.go‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,10 @@ func (p *Packet) WriteHeaderTo(buf *bytes.Buffer) {
5959
buf.Write((*[PacketHeadNoCRCLen]byte)(
6060
(unsafe.Pointer)(p),
6161
)[:])
62-
b := pbuf.NewBytes(buf.Len())
63-
b.V(func(b []byte) {
64-
copy(b, buf.Bytes())
65-
ClearTTL(b)
66-
p.md5h8 = algo.MD5Hash8(b)
67-
})
68-
b.ManualDestroy()
62+
b := make([]byte, buf.Len())
63+
copy(b, buf.Bytes())
64+
ClearTTL(b)
65+
p.md5h8 = algo.MD5Hash8(b)
6966
_ = binary.Write(buf, binary.LittleEndian, p.md5h8)
7067
return
7168
}
@@ -79,13 +76,10 @@ func (p *Packet) WriteHeaderTo(buf *bytes.Buffer) {
7976
w.Write(p.src[:])
8077
w.Write(p.dst[:])
8178
w.P(func(buf *pbuf.Buffer) {
82-
b := pbuf.NewBytes(buf.Len())
83-
b.V(func(b []byte) {
84-
copy(b, buf.Bytes())
85-
ClearTTL(b)
86-
p.md5h8 = algo.MD5Hash8(b)
87-
})
88-
b.ManualDestroy()
79+
b := make([]byte, buf.Len())
80+
copy(b, buf.Bytes())
81+
ClearTTL(b)
82+
p.md5h8 = algo.MD5Hash8(b)
8983
})
9084
w.WriteUInt64(p.md5h8)
9185
w.P(func(b *pbuf.Buffer) {

‎gold/head/builder.go‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -87,11 +87,10 @@ func (pb *DataBuilder) Zstd() *DataBuilder {
8787
return pb.p(func(ub *PacketBuf) {
8888
data := algo.EncodeZstd(ub.Bytes())
8989
ub.Reset()
90-
data.V(func(b []byte) { ub.Write(b) })
90+
ub.Write(data)
9191
if config.ShowDebugLog {
9292
logrus.Debugln(file.Header(), strconv.FormatUint(ub.DAT.md5h8, 16), "data after zstd", file.ToLimitHexString(ub.Bytes(), 64))
9393
}
94-
data.ManualDestroy()
9594
})
9695
}
9796

@@ -125,8 +124,7 @@ func (pb *DataBuilder) Seal(aead cipher.AEAD, teatyp uint8, additional uint16) *
125124
w.P(func(b *pbuf.Buffer) {
126125
data := algo.EncodeAEAD(aead, additional, b.Bytes())
127126
ub.Reset()
128-
data.V(func(b []byte) { ub.Write(b) })
129-
data.ManualDestroy()
127+
ub.Write(data)
130128
})
131129
w.Destroy()
132130
}))
@@ -204,7 +202,7 @@ func (pb *PacketBuilder) Split(mtu int, nofrag bool) (pbs []PacketBytes) {
204202
pbs = []PacketBytes{
205203
pbuf.BufferItemToBytes((*PacketItem)(
206204
pb.copy().noFrag(nofrag).hasMore(false).offset(0),
207-
)),
205+
)).Ignore(),
208206
}
209207
return
210208
}
@@ -226,7 +224,7 @@ func (pb *PacketBuilder) Split(mtu int, nofrag bool) (pbs []PacketBytes) {
226224
}
227225
pbs[i] = pbuf.BufferItemToBytes((*PacketItem)(
228226
pb.copy().offset(uint16(i*datalim)),
229-
)).Slice(a, b)
227+
)).Ignore().Slice(a, b)
230228
}
231229
})
232230
return

‎gold/head/pool.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,10 @@ import (
1212
var packetPool = pbuf.NewBufferPool[Packet]()
1313

1414
func init() {
15+
packetPool.LimitInput(256)
16+
packetPool.LimitOutput(256)
17+
pbuf.LimitInput(256)
18+
pbuf.LimitOutput(256)
1519
if config.ShowDebugLog {
1620
go status()
1721
}
@@ -23,7 +27,7 @@ func selectPacket(buf ...byte) *PacketItem {
2327
}
2428

2529
func status() {
26-
for range time.NewTicker(time.Minute).C {
30+
for range time.NewTicker(time.Second).C {
2731
out, in := packetPool.CountItems()
2832
logrus.Infoln(file.Header(), "packet outside:", out, "inside:", in)
2933
out, in = pbuf.CountItems()

‎gold/head/unbox.go‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -55,13 +55,10 @@ func ParsePacketHeader(data []byte) (pbytes PacketBytes, err error) {
5555
return
5656
}
5757
var crc uint64
58-
b := pbuf.NewBytes(int(PacketHeadNoCRCLen))
59-
b.V(func(b []byte) {
60-
copy(b, data[:PacketHeadNoCRCLen])
61-
ClearTTL(b)
62-
crc = algo.MD5Hash8(b)
63-
})
64-
b.ManualDestroy()
58+
var b [PacketHeadNoCRCLen]byte
59+
copy(b[:], data[:PacketHeadNoCRCLen])
60+
ClearTTL(b[:])
61+
crc = algo.MD5Hash8(b[:])
6562
if crc != pb.DAT.md5h8 {
6663
err = ErrBadCRCChecksum
6764
if config.ShowDebugLog {
@@ -85,7 +82,7 @@ func ParsePacketHeader(data []byte) (pbytes PacketBytes, err error) {
8582
p.ManualDestroy()
8683
return
8784
}
88-
pbytes = pbuf.BufferItemToBytes(p)
85+
pbytes = pbuf.BufferItemToBytes(p).Ignore()
8986
return
9087
}
9188

‎gold/link/crypto.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package link
33
import (
44
"encoding/hex"
55

6-
"github.com/fumiama/orbyte/pbuf"
76
"github.com/sirupsen/logrus"
87

98
"github.com/fumiama/WireGold/config"
@@ -19,7 +18,7 @@ func (l *Link) randkeyidx() uint8 {
1918
}
2019

2120
// decode by aead and put b into pool
22-
func (l *Link) decode(teatype uint8, additional uint16, b []byte) (db pbuf.Bytes, err error) {
21+
func (l *Link) decode(teatype uint8, additional uint16, b []byte) (db []byte, err error) {
2322
if len(b) == 0 || teatype >= 32 {
2423
return
2524
}
@@ -33,7 +32,9 @@ func (l *Link) decode(teatype uint8, additional uint16, b []byte) (db pbuf.Bytes
3332
}
3433
logrus.Debugln(file.Header(), "copy plain text", hex.EncodeToString(b[:n]), endl)
3534
}
36-
return pbuf.ParseBytes(b...).Copy(), nil
35+
db = make([]byte, len(b))
36+
copy(db, b)
37+
return
3738
}
3839
aead := l.keys[teatype]
3940
if aead == nil {
@@ -43,7 +44,7 @@ func (l *Link) decode(teatype uint8, additional uint16, b []byte) (db pbuf.Bytes
4344
}
4445

4546
// xorenc 按 8 字节, 以初始 m.mask 循环异或编码 data
46-
func (m *Me) xorenc(data []byte, seq uint32) pbuf.Bytes {
47+
func (m *Me) xorenc(data []byte, seq uint32) []byte {
4748
return algo.EncodeXOR(data, m.mask, seq)
4849
}
4950

‎gold/link/event.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,12 @@ package link
33
import (
44
"github.com/RomiChan/syncx"
55
"github.com/fumiama/WireGold/gold/head"
6-
"github.com/fumiama/orbyte/pbuf"
76
)
87

98
// 事件分发器
109
var dispachers syncx.Map[uint8, Dispacher]
1110

12-
type Dispacher func(header *head.Packet, peer *Link, data pbuf.Bytes)
11+
type Dispacher func(header *head.Packet, peer *Link, data []byte)
1312

1413
// RegisterDispacher of proto
1514
func RegisterDispacher(p uint8, d Dispacher) (actual Dispacher, hasexist bool) {

‎gold/link/link.go‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import (
1212
"github.com/fumiama/WireGold/gold/p2p"
1313
"github.com/fumiama/WireGold/internal/bin"
1414
base14 "github.com/fumiama/go-base16384"
15-
"github.com/fumiama/orbyte/pbuf"
1615
"github.com/sirupsen/logrus"
1716
)
1817

@@ -70,25 +69,24 @@ func (m *Me) Connect(peer string) (*Link, error) {
7069
return nil, ErrPerrNotExist
7170
}
7271

73-
func (l *Link) ToLower(header *head.Packet, data pbuf.Bytes) {
72+
func (l *Link) ToLower(header *head.Packet, data []byte) {
7473
if l.pipe != nil {
74+
d := make([]byte, len(data))
75+
copy(d, data)
7576
l.pipe <- LinkData{
7677
H: *header,
77-
D: data.Copy().Trans(),
78+
D: d,
7879
}
7980
if config.ShowDebugLog {
8081
logrus.Debugln("[listen] deliver to pipe of", l.peerip)
8182
}
8283
return
8384
}
84-
var err error
85-
data.V(func(b []byte) {
86-
_, err = l.me.nic.Write(b)
87-
})
85+
_, err := l.me.nic.Write(data)
8886
if err != nil {
89-
logrus.Errorln("[listen] deliver", data.Len(), "bytes data to nic err:", err)
87+
logrus.Errorln("[listen] deliver", len(data), "bytes data to nic err:", err)
9088
} else if config.ShowDebugLog {
91-
logrus.Debugln("[listen] deliver", data.Len(), "bytes data to nic")
89+
logrus.Debugln("[listen] deliver", len(data), "bytes data to nic")
9290
}
9391
}
9492

‎gold/link/listen.go‎

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,17 @@ func (m *Me) runworkers() {
3232
m.jobs = make([]chan job, ncpu)
3333
for i := 0; i < ncpu; i++ {
3434
m.jobs[i] = make(chan job, 4096)
35-
go func(jobs <-chan job) {
35+
go func(i int, jobs <-chan job) {
3636
for jb := range jobs {
37+
if config.ShowDebugLog {
38+
logrus.Debugln("[listen] job thread", i, "call waitordispatch")
39+
}
3740
m.waitordispatch(jb.addr, jb.buf, jb.n, jb.fil)
41+
if config.ShowDebugLog {
42+
logrus.Debugln("[listen] job thread", i, "fin waitordispatch")
43+
}
3844
}
39-
}(m.jobs[i])
45+
}(i, m.jobs[i])
4046
}
4147
}
4248

@@ -73,9 +79,9 @@ func (m *Me) listen() (conn p2p.Conn, err error) {
7379
fil *uintptr
7480
)
7581
if idx < 0 {
76-
lbf = pbuf.NewBytes(lstnbufgragsz)
82+
lbf = pbuf.NewLargeBytes(lstnbufgragsz)
7783
} else {
78-
lbf = pbuf.ParseBytes(bufs[idx*lstnbufgragsz : (idx+1)*lstnbufgragsz]...)
84+
lbf = pbuf.ParseBytes(bufs[idx*lstnbufgragsz : (idx+1)*lstnbufgragsz : (idx+1)*lstnbufgragsz]...).Ignore()
7985
fil = &fils[idx]
8086
}
8187

@@ -194,37 +200,31 @@ func (m *Me) dispatch(header *head.Packet, body []byte, addr p2p.EndPoint) {
194200
}
195201
return
196202
}
197-
if data.Len() < 8 {
203+
if len(data) < 8 {
198204
if config.ShowDebugLog {
199-
logrus.Debugln("[listen] drop invalid data len packet key idx:", header.CipherIndex(), "addt:", addt, "len", data.Len())
205+
logrus.Debugln("[listen] drop invalid data len packet key idx:", header.CipherIndex(), "addt:", addt, "len", len(data))
200206
}
201207
return
202208
}
203209
ok := false
204-
data.V(func(b []byte) {
205-
ok = algo.IsVaildBlake2bHash8(header.PreCRC64(), b)
206-
})
210+
ok = algo.IsVaildBlake2bHash8(header.PreCRC64(), data)
207211
if !ok {
208212
if config.ShowDebugLog {
209213
logrus.Debugln("[listen] drop invalid hash packet")
210214
}
211215
return
212216
}
213-
data = data.SliceFrom(8)
217+
data = data[8:]
214218
if p.usezstd {
215-
data.V(func(b []byte) {
216-
old := data
217-
data, err = algo.DecodeZstd(b) // skip hash
218-
old.ManualDestroy()
219-
})
219+
data, err = algo.DecodeZstd(data) // skip hash
220220
if err != nil {
221221
if config.ShowDebugLog {
222222
logrus.Debugln("[listen] drop invalid zstd packet:", err)
223223
}
224224
return
225225
}
226226
if config.ShowDebugLog {
227-
logrus.Debugln("[listen] zstd decoded len:", data.Len())
227+
logrus.Debugln("[listen] zstd decoded len:", len(data))
228228
}
229229
}
230230
fn, ok := GetDispacher(header.Proto.Proto())
@@ -233,5 +233,4 @@ func (m *Me) dispatch(header *head.Packet, body []byte, addr p2p.EndPoint) {
233233
return
234234
}
235235
fn(header, p, data)
236-
data.ManualDestroy()
237236
}

0 commit comments

Comments
 (0)