Skip to content

Commit ffefd96

Browse files
alex-aizmanalex-aizman
committed
feat: native bucket inventory (remove page-based knobs; fix)
* replace PagesPerChunk / MaxEntriesPerChunk with a single NamesPerChunk * remove page-count–based chunking logic - chunk fill now driven purely by entry count * revise NBI listing continuation handling - simplify token emission: non-empty page => last name * tests: - update to NamesPerChunk - allow empty inventory name (default selection) * misc: - fix NBIInfoMap.SingleName() to return actual name - minor logging and naming cleanups ---- * part twenty-nine, prev. commit: 3674366 Signed-off-by: Alex Aizman <alex.aizman@gmail.com>
1 parent a8dabc5 commit ffefd96

9 files changed

Lines changed: 107 additions & 212 deletions

File tree

‎ais/test/nbi_test.go‎

Lines changed: 49 additions & 136 deletions
Large diffs are not rendered by default.

‎api/apc/nbi.go‎

Lines changed: 26 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,10 @@ type (
1818
Name string `json:"name,omitempty"` // inventory name (optional; must be unique for a given bucket)
1919
LsoMsg
2020

21-
// PagesPerChunk overrides the default number of pages to pack into a single inventory chunk.
22-
// If zero, the default is used.
23-
PagesPerChunk int64 `json:"pages_per_chunk,omitempty"`
24-
25-
// MaxEntriesPerChunk puts a hard cap on the number of entries in a single inventory chunk.
26-
// If zero, the cap is disabled.
27-
MaxEntriesPerChunk int64 `json:"max_entries_per_chunk,omitempty"`
21+
// Number of object names to store in each inventory chunk.
22+
// Requested properties are stored alongside each name.
23+
// Advanced usage only - non-zero overrides system default.
24+
NamesPerChunk int64 `json:"names_per_chunk,omitempty"`
2825

2926
// Remove all existing inventories, if any, and proceed to create the new one
3027
// (only one inventory per bucket is supported).
@@ -48,11 +45,11 @@ type (
4845
//
4946

5047
func (m *LsoMsg) ValidateNBI() error {
51-
const etag = "invalid list via native bucket inventory"
48+
const epref = "invalid list via native bucket inventory"
5249

5350
// inventory snapshot is flat; StartAfter currently unsupported
5451
if m.StartAfter != "" {
55-
return errors.New(etag + ": start_after is not supported")
52+
return errors.New(epref + ": start_after is not supported")
5653
}
5754

5855
// flags that do not make sense for inventory listing
@@ -64,7 +61,7 @@ func (m *LsoMsg) ValidateNBI() error {
6461
sb.Grow(96)
6562
sb.WriteString("flags:")
6663
m.appendFlags(&sb)
67-
return fmt.Errorf("%s: %s", etag, sb.String())
64+
return fmt.Errorf("%s: %s", epref, sb.String())
6865
}
6966

7067
return nil
@@ -75,21 +72,21 @@ func (m *LsoMsg) ValidateNBI() error {
7572
//////////////////
7673

7774
const (
78-
DefaultInvPagesPerChunk = 50
79-
MaxInvPagesPerChunk = 256
80-
MaxInvEntriesPerChunk = MaxInvPagesPerChunk * MaxPageSizeGlobal
75+
DfltInvNamesPerChunk = 2 * MaxPageSizeAIS // 20K
76+
MaxInvNamesPerChunk = 64 * MaxPageSizeAIS // 640K
77+
MinInvNamesPerChunk = 2
8178
)
8279

8380
// validate; set defaults
8481
func (m *CreateNBIMsg) SetValidate() error {
85-
const etag = "invalid '" + ActCreateNBI + "'"
82+
const epref = "invalid '" + ActCreateNBI + "'"
8683

8784
// 1) disallow
8885
if m.ContinuationToken != "" {
89-
return errors.New(etag + ": continuation_token must be empty")
86+
return errors.New(epref + ": continuation_token must be empty")
9087
}
9188
if m.StartAfter != "" {
92-
return errors.New(etag + ": start_after is not supported")
89+
return errors.New(epref + ": start_after is not supported")
9390
}
9491
// flags that don't make sense for inventory generation
9592
const badFlags = LsCached | LsNotCached | LsMissing | LsDeleted | LsArchDir |
@@ -100,23 +97,19 @@ func (m *CreateNBIMsg) SetValidate() error {
10097
sb.Grow(96)
10198
sb.WriteString("flags:")
10299
m.appendFlags(&sb)
103-
return fmt.Errorf("%s: %s", etag, sb.String())
100+
return fmt.Errorf("%s: %s", epref, sb.String())
104101
}
105102

106103
// 2) advanced tunables
107104
switch {
108-
case m.PagesPerChunk == 0:
109-
m.PagesPerChunk = DefaultInvPagesPerChunk
110-
case m.PagesPerChunk < 0:
111-
return fmt.Errorf("%s: pages_per_chunk=%d", etag, m.PagesPerChunk)
112-
case m.PagesPerChunk > MaxInvPagesPerChunk:
113-
return fmt.Errorf("%s: pages_per_chunk too large: %d", etag, m.PagesPerChunk)
114-
}
115-
if m.MaxEntriesPerChunk < 0 {
116-
return fmt.Errorf("%s: negative max_entries_per_chunk=%d", etag, m.MaxEntriesPerChunk)
117-
}
118-
if m.MaxEntriesPerChunk > MaxInvEntriesPerChunk {
119-
return fmt.Errorf("%s: too large max_entries_per_chunk=%d", etag, m.MaxEntriesPerChunk)
105+
case m.NamesPerChunk == 0:
106+
m.NamesPerChunk = DfltInvNamesPerChunk
107+
case m.NamesPerChunk < 0:
108+
return fmt.Errorf("%s: negative names_per_chunk=%d", epref, m.NamesPerChunk)
109+
case m.NamesPerChunk < MinInvNamesPerChunk:
110+
return fmt.Errorf("%s: names_per_chunk=%d too small (min=%d)", epref, m.NamesPerChunk, MinInvNamesPerChunk)
111+
case m.NamesPerChunk > MaxInvNamesPerChunk:
112+
return fmt.Errorf("%s: names_per_chunk=%d too large (max=%d)", epref, m.NamesPerChunk, MaxInvNamesPerChunk)
120113
}
121114

122115
// 3) NOTE: otherwise, backend _may_ append extra (virt-dir) entries (in re: pre-allocation+reuse)
@@ -158,9 +151,9 @@ func (m NBIInfoMap) Names() []string {
158151
return names
159152
}
160153

161-
func (m NBIInfoMap) SingleName() (name string) {
162-
for name = range m {
163-
break
154+
func (m NBIInfoMap) SingleName() string {
155+
for _, info := range m {
156+
return info.Name
164157
}
165-
return
158+
return ""
166159
}

‎cmd/cli/cli/const.go‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1436,13 +1436,9 @@ var (
14361436
}
14371437

14381438
// advanced options
1439-
nbiPagesPerChunkFlag = cli.IntFlag{
1440-
Name: "inv-pages",
1441-
Usage: "Number of list-object pages to pack in a single inventory chunk (advanced usage)",
1442-
}
1443-
nbiMaxEntriesPerChunkFlag = cli.IntFlag{
1444-
Name: "inv-max-entries",
1445-
Usage: "Maximum number of entries per inventory chunk (advanced usage)",
1439+
nbiNamesPerChunkFlag = cli.IntFlag{
1440+
Name: "names-per-chunk",
1441+
Usage: "Number of object names per inventory chunk (advanced usage)",
14461442
}
14471443
nbiForceFlag = cli.BoolFlag{
14481444
Name: forceFlag.Name,

‎cmd/cli/cli/nbi_hdlr.go‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,7 @@ var (
4747
invPrefixFlag,
4848
nameOnlyFlag,
4949
allPropsFlag,
50-
nbiPagesPerChunkFlag,
51-
nbiMaxEntriesPerChunkFlag,
50+
nbiNamesPerChunkFlag,
5251
nbiForceFlag,
5352
},
5453
commandRemove: {
@@ -143,13 +142,9 @@ func createNBIHandler(c *cli.Context) error {
143142
}
144143

145144
// advanced
146-
if flagIsSet(c, nbiPagesPerChunkFlag) {
147-
a := parseIntFlag(c, nbiPagesPerChunkFlag)
148-
msg.PagesPerChunk = int64(a)
149-
}
150-
if flagIsSet(c, nbiMaxEntriesPerChunkFlag) {
151-
a := parseIntFlag(c, nbiMaxEntriesPerChunkFlag)
152-
msg.MaxEntriesPerChunk = int64(a)
145+
if flagIsSet(c, nbiNamesPerChunkFlag) {
146+
a := parseIntFlag(c, nbiNamesPerChunkFlag)
147+
msg.NamesPerChunk = int64(a)
153148
}
154149
msg.Force = flagIsSet(c, nbiForceFlag)
155150

‎cmd/cli/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/cli
33
go 1.25
44

55
require (
6-
github.com/NVIDIA/aistore v1.4.3-0.20260311173632-f3472105d2ea
6+
github.com/NVIDIA/aistore v1.4.3-0.20260317160545-bf3c128925d2
77
github.com/fatih/color v1.18.0
88
github.com/json-iterator/go v1.1.12
99
github.com/onsi/ginkgo/v2 v2.27.5

‎cmd/cli/go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
22
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
33
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
4-
github.com/NVIDIA/aistore v1.4.3-0.20260311173632-f3472105d2ea h1:YBsuGoB+GBGDcCJeQAHRB3lADTKymwtiqPprLO6AjyE=
5-
github.com/NVIDIA/aistore v1.4.3-0.20260311173632-f3472105d2ea/go.mod h1:LWOYptrTZyGfxqIg6M7IIO+8UNmHBQZM68CbYVyPVM8=
4+
github.com/NVIDIA/aistore v1.4.3-0.20260317160545-bf3c128925d2 h1:viUKZ18MT0AE223hYU95kUwsYs3CoJ46Fk0K9A12cOQ=
5+
github.com/NVIDIA/aistore v1.4.3-0.20260317160545-bf3c128925d2/go.mod h1:LWOYptrTZyGfxqIg6M7IIO+8UNmHBQZM68CbYVyPVM8=
66
github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
77
github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
88
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=

‎xact/xs/create_nbi.go‎

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,6 @@ const (
4545
nbiMaxHdrLen = 4 * cos.KiB // (unlikely to ever exceed)
4646
)
4747

48-
const (
49-
nbiMinEntriesPerChunk = 2
50-
)
51-
5248
type (
5349
nbiChunkHdr struct {
5450
first string
@@ -195,30 +191,26 @@ func (r *XactNBI) Run(wg *sync.WaitGroup) {
195191
if pgsize := bck.MaxPageSize(); lsmsg.PageSize <= 0 || lsmsg.PageSize > pgsize {
196192
lsmsg.PageSize = pgsize
197193
}
198-
// adjust to min
199-
if n := r.msg.MaxEntriesPerChunk; n > 0 {
200-
r.msg.PagesPerChunk = min(cos.DivRoundI64(n, lsmsg.PageSize), r.msg.PagesPerChunk)
201-
}
202194
lsmsg.ContinuationToken = ""
203195

204196
var (
197+
ntotal int64
205198
smap = core.T.Sowner().Get()
206199
bp = core.T.Backend(bck)
207200
ubuf = bck.MakeUname("", true)
208201
lastToken = "__dummy__"
209-
all = make(cmn.LsoEntries, 0, lsmsg.PageSize*r.msg.PagesPerChunk) // prealloc and reuse
202+
all = make(cmn.LsoEntries, 0, r.msg.NamesPerChunk) // prealloc and reuse
210203
fast = true
211204
nonRecurs = lsmsg.IsFlagSet(apc.LsNoRecursion)
212205
)
213206
const warn = "backend returned non-reused entries slice; falling back to append path"
214207
for num := 1; !r.IsAborted() && lastToken != ""; num++ {
215208
var (
216-
idx int
217-
npages int64
209+
idx int
218210
)
219211
all = all[:0]
220212

221-
for ; lastToken != "" && (npages < r.msg.PagesPerChunk || idx < nbiMinEntriesPerChunk); npages++ {
213+
for lastToken != "" && idx < int(r.msg.NamesPerChunk) {
222214
lst := &cmn.LsoRes{}
223215
dst := all[idx:idx:cap(all)] // safe for append
224216
lst.Entries = dst
@@ -246,8 +238,8 @@ func (r *XactNBI) Run(wg *sync.WaitGroup) {
246238
switch {
247239
case fast && !reused: // switch fast => slow
248240
fast = false
249-
nlog.Errorln(r.Name(), "Warning: fast->slow:", warn) // TODO: find out
250-
tmp := make(cmn.LsoEntries, 0, lsmsg.PageSize*r.msg.PagesPerChunk)
241+
nlog.Warningln(r.Name(), "fast->slow:", warn) // TODO: find out
242+
tmp := make(cmn.LsoEntries, 0, r.msg.NamesPerChunk)
251243
tmp = append(tmp, all[:idx]...)
252244
tmp = append(tmp, lst.Entries...)
253245
all = tmp
@@ -272,16 +264,17 @@ func (r *XactNBI) Run(wg *sync.WaitGroup) {
272264

273265
// next chunk
274266
all = all[:idx]
267+
ntotal += int64(idx)
275268

276269
if err := r.writeChunk(num, all); err != nil {
277270
r.Abort(err)
278271
return
279272
}
280273
if cmn.Rom.V(5, cos.ModXs) {
281274
if idx == 1 {
282-
nlog.Warningln(core.T.String(), r.Name(), "single-entry chunk-num:", num, "npages:", npages)
275+
nlog.Warningln(core.T.String(), r.Name(), "single-entry chunk-num:", num, "total:", ntotal)
283276
} else {
284-
nlog.Infoln(r.Name(), "chunk-num:", num, "entries:", idx, "npages:", npages)
277+
nlog.Infoln(r.Name(), "chunk-num:", num, "lso-entries:", idx, "total:", ntotal)
285278
}
286279
}
287280
}
@@ -303,6 +296,11 @@ func (r *XactNBI) Run(wg *sync.WaitGroup) {
303296
core.T.FSHC(err, r.lom.Mountpath(), r.lom.FQN)
304297
}
305298

299+
if cmn.Rom.V(5, cos.ModXs) {
300+
nlog.Infof("completed %s [%s]: size %d, chunks %d, lso-entries %d (per chunk: %d)",
301+
r.lom.Cname(), r.msg.Name, r.lom.Lsize(), r.ufest.Count(), ntotal, r.msg.NamesPerChunk)
302+
}
303+
306304
r.cleanup()
307305
r.Finish()
308306
}

‎xact/xs/ls_nbi.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -275,16 +275,15 @@ func (nbi *nbiCtx) nextPage(msg *apc.LsoMsg, lst *cmn.LsoRes) error {
275275
lst.Entries = append(lst.Entries, out)
276276
}
277277

278-
more := len(nbi.entries) > 0
279-
if nbi.nidx == len(nbi.entries) {
280-
more = nbi.chunkNum < nbi.ufest.Count()
281-
}
282-
if l := len(lst.Entries); more && l > 0 {
278+
if l := len(lst.Entries); l > 0 {
283279
lst.ContinuationToken = lst.Entries[l-1].Name
284280
debug.Assert(lst.ContinuationToken != "")
285281

286-
// out of prefix-defined range
287-
if msg.Prefix != "" {
282+
// eof
283+
if nbi.nidx >= len(nbi.entries) && nbi.chunkNum > nbi.ufest.Count() {
284+
lst.ContinuationToken = ""
285+
} else if msg.Prefix != "" {
286+
// out of prefix-defined range
288287
if nbi.nidx < len(nbi.entries) && !strings.HasPrefix(nbi.entries[nbi.nidx].Name, msg.Prefix) {
289288
lst.ContinuationToken = ""
290289
}
@@ -302,6 +301,7 @@ func (nbi *nbiCtx) nextPage(msg *apc.LsoMsg, lst *cmn.LsoRes) error {
302301

303302
func (nbi *nbiCtx) seekAfter(token string) error {
304303
debug.Assert(token != "")
304+
debug.Assert(nbi.nidx == 0) // otherwise we better search from nidx onwards
305305

306306
// fast path: search current chunk
307307
if nbi.nidx < len(nbi.entries) && len(nbi.entries) > 0 && token < nbi.hdr.last {

‎xact/xs/lso.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ func (p *lsoFactory) Start() error {
180180

181181
if r.msg.IsFlagSet(apc.LsNBI) {
182182
invName := p.hdr.Get(apc.HdrInvName)
183-
debug.Assert(invName != "") // checked by target
183+
debug.Assert(invName != "") // checked (or set) by target
184184
r.nbi = &nbiCtx{bck: bck}
185185
if err := r.nbi.init(invName); err != nil {
186186
return err

0 commit comments

Comments
 (0)