Skip to content

Commit 5574e4f

Browse files
committed
util: add GroupLock for two-group mutual exclusion
Add GroupLock type that allows operations within the same group to run concurrently while blocking operations from the other group. This wiil be in used to coordinate NVMe-oF Stage and Unstage operations - multiple Stage calls can run in parallel, multiple Unstage calls can run in parallel, but Stage and Unstage block each other. This can be in used in any place which needs lock between 2 types of group. The implementation is based on Keane and Moir's "group mutual exclusion algorithm", but simplified for exactly two groups using Golang condition variables. Operations in group A wait if any group B operations are active, and vice versa. When the last operation in a group completes,all waiting operations in the other group are woken up together. Also added some test, simulate parallel workers: - Multiple same-group operations run concurrently - Different groups block each other - No deadlock under heavy contention - Mutual exclusion is never violated - All waiting operations start together when unblocked Signed-off-by: gadi-didi <gadi.didi@ibm.com>
1 parent 87131da commit 5574e4f

File tree

2 files changed

+555
-0
lines changed

2 files changed

+555
-0
lines changed

internal/util/lock/group_lock.go

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/*
2+
Copyright 2026 The Ceph-CSI Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package lock
18+
19+
import "sync"
20+
21+
// GroupLock implements a simplified version of Group Mutual Exclusion
22+
// for exactly 2 groups. For the general N-group case see:
23+
//
24+
// - Keane, P. and Moir, M. - "A simple local-spin group mutual exclusion algorithm"
25+
// - Link: https://dl.acm.org/doi/epdf/10.1145/301308.301319
26+
//
27+
// GroupLock implements mutual exclusion between two groups of operations.
28+
// Operations within the same group can run concurrently, but operations
29+
// from different groups cannot run simultaneously.
30+
//
31+
// Example use cases:
32+
// - Allowing multiple Stage operations OR multiple Unstage operations, but not both
33+
// - Allowing multiple Read operations OR multiple Write operations, but not both
34+
// - Allowing multiple Connect operations OR multiple Disconnect operations, but not both
35+
//
36+
// This is sometimes called "symmetric reader-writer lock" or "group mutual exclusion".
37+
//
38+
// NOTE:
39+
// GroupLock does not guarantee fairness. Under heavy load from one group,
40+
// the other group may experience temporary delays!!
41+
type GroupLock struct {
42+
mutex sync.Mutex
43+
groupACount int // Number of active Group A operations
44+
groupBCount int // Number of active Group B operations
45+
groupACond *sync.Cond // Signals when groupBCount becomes 0
46+
groupBCond *sync.Cond // Signals when groupACount becomes 0
47+
}
48+
49+
// NewGroupLock creates a new GroupLock.
50+
func NewGroupLock() *GroupLock {
51+
gl := &GroupLock{}
52+
gl.groupACond = sync.NewCond(&gl.mutex)
53+
gl.groupBCond = sync.NewCond(&gl.mutex)
54+
55+
return gl
56+
}
57+
58+
// AcquireGroupA acquires the lock for a Group A operation.
59+
// Multiple Group A operations can proceed concurrently, but they will block
60+
// if any Group B operations are active.
61+
func (gl *GroupLock) AcquireGroupA() {
62+
gl.mutex.Lock()
63+
defer gl.mutex.Unlock()
64+
65+
// Wait while any Group B operations are active
66+
for gl.groupBCount > 0 {
67+
// Wait releases the mutex and blocks until Broadcast is called, then re-acquires the mutex before returning
68+
gl.groupACond.Wait()
69+
}
70+
71+
gl.groupACount++
72+
}
73+
74+
// ReleaseGroupA releases the lock for a Group A operation.
75+
func (gl *GroupLock) ReleaseGroupA() {
76+
gl.mutex.Lock()
77+
defer gl.mutex.Unlock()
78+
79+
gl.groupACount--
80+
81+
// If this was the last Group A operation, wake up waiting Group B operations
82+
if gl.groupACount == 0 {
83+
gl.groupBCond.Broadcast()
84+
}
85+
}
86+
87+
// AcquireGroupB acquires the lock for a Group B operation.
88+
// Multiple Group B operations can proceed concurrently, but they will block
89+
// if any Group A operations are active.
90+
func (gl *GroupLock) AcquireGroupB() {
91+
gl.mutex.Lock()
92+
defer gl.mutex.Unlock()
93+
94+
// Wait while any Group A operations are active
95+
for gl.groupACount > 0 {
96+
// Wait releases the mutex and blocks until Broadcast is called, then re-acquires the mutex before returning
97+
gl.groupBCond.Wait()
98+
}
99+
100+
gl.groupBCount++
101+
}
102+
103+
// ReleaseGroupB releases the lock for a Group B operation.
104+
func (gl *GroupLock) ReleaseGroupB() {
105+
gl.mutex.Lock()
106+
defer gl.mutex.Unlock()
107+
108+
gl.groupBCount--
109+
110+
// If this was the last Group B operation, wake up waiting Group A operations
111+
if gl.groupBCount == 0 {
112+
gl.groupACond.Broadcast()
113+
}
114+
}

0 commit comments

Comments
 (0)