Skip to content

Commit 6afe68a

Browse files
erizocosmicoerizocosmico
committed
gitbase: support for unpacked objects, integration tests added
This PR adds support for unpacked objects, which made the index contain different objects and not match the result of the query without indexes. A bunch of integration tests for indexes of all tables testing they output the exact same thing as the same query without indexes have been added as well. Because it's super important to ensure the index is closed so it can be released, each index iterator has now a call to `closeIndexOnError`, which closes the index if there is an error or EOF in an iterator, ensure it's *always* closed. Signed-off-by: Miguel Molina <miguel@erizocosmi.co>
1 parent 492ae11 commit 6afe68a

File tree

9 files changed

+630
-393
lines changed

9 files changed

+630
-393
lines changed

‎blobs.go‎

Lines changed: 91 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ func (*blobsTable) IndexKeyValueIter(
128128
return nil, ErrInvalidGitbaseSession.New(ctx.Session)
129129
}
130130

131-
return newBlobsKeyValueIter(s.Pool, colNames), nil
131+
return newBlobsKeyValueIter(s.Pool, colNames)
132132
}
133133

134134
// WithProjectFiltersAndIndex implements sql.Indexable interface.
@@ -149,11 +149,7 @@ func (*blobsTable) WithProjectFiltersAndIndex(
149149
return nil, err
150150
}
151151

152-
var iter sql.RowIter = &blobsIndexIter{
153-
index: index,
154-
pool: session.Pool,
155-
readContent: shouldReadContent(columns),
156-
}
152+
var iter sql.RowIter = newBlobsIndexIter(index, session.Pool, shouldReadContent(columns))
157153

158154
if len(filters) > 0 {
159155
iter = plan.NewFilterIter(ctx, expression.JoinAnd(filters...), iter)
@@ -337,61 +333,112 @@ func shouldReadContent(columns []sql.Expression) bool {
337333
}
338334

339335
type blobsKeyValueIter struct {
340-
iter *objectIter
336+
pool *RepositoryPool
337+
repos *RepositoryIter
338+
repo *Repository
339+
blobs *object.BlobIter
340+
idx *repositoryIndex
341341
columns []string
342342
}
343343

344-
func newBlobsKeyValueIter(pool *RepositoryPool, columns []string) *blobsKeyValueIter {
344+
func newBlobsKeyValueIter(pool *RepositoryPool, columns []string) (*blobsKeyValueIter, error) {
345+
repos, err := pool.RepoIter()
346+
if err != nil {
347+
return nil, err
348+
}
349+
345350
return &blobsKeyValueIter{
346-
iter: newObjectIter(pool, plumbing.BlobObject),
351+
pool: pool,
352+
repos: repos,
347353
columns: columns,
348-
}
354+
}, nil
349355
}
350356

351357
func (i *blobsKeyValueIter) Next() ([]interface{}, []byte, error) {
352-
obj, err := i.iter.Next()
353-
if err != nil {
354-
return nil, nil, err
355-
}
358+
for {
359+
if i.blobs == nil {
360+
var err error
361+
i.repo, err = i.repos.Next()
362+
if err != nil {
363+
return nil, nil, err
364+
}
356365

357-
key, err := encodeIndexKey(packOffsetIndexKey{
358-
Repository: obj.RepositoryID,
359-
Packfile: obj.Packfile.String(),
360-
Offset: int64(obj.Offset),
361-
})
362-
if err != nil {
363-
return nil, nil, err
364-
}
366+
i.blobs, err = i.repo.Repo.BlobObjects()
367+
if err != nil {
368+
return nil, nil, err
369+
}
365370

366-
blob, ok := obj.Object.(*object.Blob)
367-
if !ok {
368-
ErrInvalidObjectType.New(obj.Object, "*object.Blob")
369-
}
371+
repo := i.pool.repositories[i.repo.ID]
372+
i.idx, err = newRepositoryIndex(repo.path, repo.kind)
373+
if err != nil {
374+
return nil, nil, err
375+
}
376+
}
370377

371-
row, err := blobToRow(obj.RepositoryID, blob, stringContains(i.columns, "blob_content"))
372-
if err != nil {
373-
return nil, nil, err
374-
}
378+
blob, err := i.blobs.Next()
379+
if err != nil {
380+
if err == io.EOF {
381+
i.blobs = nil
382+
continue
383+
}
384+
}
375385

376-
values, err := rowIndexValues(row, i.columns, BlobsSchema)
377-
if err != nil {
378-
return nil, nil, err
379-
}
386+
offset, packfile, err := i.idx.find(blob.Hash)
387+
if err != nil {
388+
return nil, nil, err
389+
}
390+
391+
var hash string
392+
if offset < 0 {
393+
hash = blob.Hash.String()
394+
}
395+
396+
key, err := encodeIndexKey(packOffsetIndexKey{
397+
Repository: i.repo.ID,
398+
Packfile: packfile.String(),
399+
Offset: offset,
400+
Hash: hash,
401+
})
402+
if err != nil {
403+
return nil, nil, err
404+
}
380405

381-
return values, key, nil
406+
row, err := blobToRow(i.repo.ID, blob, stringContains(i.columns, "blob_content"))
407+
if err != nil {
408+
return nil, nil, err
409+
}
410+
411+
values, err := rowIndexValues(row, i.columns, BlobsSchema)
412+
if err != nil {
413+
return nil, nil, err
414+
}
415+
416+
return values, key, nil
417+
}
382418
}
383419

384-
func (i *blobsKeyValueIter) Close() error { return i.iter.Close() }
420+
func (i *blobsKeyValueIter) Close() error { return i.repos.Close() }
385421

386422
type blobsIndexIter struct {
387423
index sql.IndexValueIter
388-
pool *RepositoryPool
389424
decoder *objectDecoder
390425
readContent bool
391426
}
392427

428+
func newBlobsIndexIter(index sql.IndexValueIter, pool *RepositoryPool, readContent bool) *blobsIndexIter {
429+
return &blobsIndexIter{
430+
index: index,
431+
decoder: newObjectDecoder(pool),
432+
readContent: readContent,
433+
}
434+
}
435+
393436
func (i *blobsIndexIter) Next() (sql.Row, error) {
394-
data, err := i.index.Next()
437+
var err error
438+
var data []byte
439+
defer closeIndexOnError(&err, i.index)
440+
441+
data, err = i.index.Next()
395442
if err != nil {
396443
return nil, err
397444
}
@@ -401,21 +448,12 @@ func (i *blobsIndexIter) Next() (sql.Row, error) {
401448
return nil, err
402449
}
403450

404-
packfile := plumbing.NewHash(key.Packfile)
405-
if i.decoder == nil || !i.decoder.equals(key.Repository, packfile) {
406-
if i.decoder != nil {
407-
if err := i.decoder.Close(); err != nil {
408-
return nil, err
409-
}
410-
}
411-
412-
i.decoder, err = newObjectDecoder(i.pool.repositories[key.Repository], packfile)
413-
if err != nil {
414-
return nil, err
415-
}
416-
}
417-
418-
obj, err := i.decoder.get(key.Offset)
451+
obj, err := i.decoder.decode(
452+
key.Repository,
453+
plumbing.NewHash(key.Packfile),
454+
key.Offset,
455+
plumbing.NewHash(key.Hash),
456+
)
419457
if err != nil {
420458
return nil, err
421459
}

‎commits.go‎

Lines changed: 90 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ func (*commitsTable) IndexKeyValueIter(
119119
return nil, ErrInvalidGitbaseSession.New(ctx.Session)
120120
}
121121

122-
return newCommitsKeyValueIter(s.Pool, colNames), nil
122+
return newCommitsKeyValueIter(s.Pool, colNames)
123123
}
124124

125125
// WithProjectFiltersAndIndex implements sql.Indexable interface.
@@ -140,7 +140,7 @@ func (*commitsTable) WithProjectFiltersAndIndex(
140140
return nil, err
141141
}
142142

143-
var iter sql.RowIter = &commitsIndexIter{index: index, pool: session.Pool}
143+
var iter sql.RowIter = newCommitsIndexIter(index, session.Pool)
144144

145145
if len(filters) > 0 {
146146
iter = plan.NewFilterIter(ctx, expression.JoinAnd(filters...), iter)
@@ -298,80 +298,123 @@ func (i *commitsByHashIter) nextList() (*object.Commit, error) {
298298
}
299299

300300
type commitsKeyValueIter struct {
301-
iter *objectIter
301+
repo *Repository
302+
pool *RepositoryPool
303+
repos *RepositoryIter
304+
commits object.CommitIter
305+
idx *repositoryIndex
302306
columns []string
303307
}
304308

305-
func newCommitsKeyValueIter(pool *RepositoryPool, columns []string) *commitsKeyValueIter {
309+
func newCommitsKeyValueIter(pool *RepositoryPool, columns []string) (*commitsKeyValueIter, error) {
310+
repos, err := pool.RepoIter()
311+
if err != nil {
312+
return nil, err
313+
}
314+
306315
return &commitsKeyValueIter{
307-
iter: newObjectIter(pool, plumbing.CommitObject),
316+
pool: pool,
317+
repos: repos,
308318
columns: columns,
309-
}
319+
}, nil
310320
}
311321

312322
func (i *commitsKeyValueIter) Next() ([]interface{}, []byte, error) {
313-
obj, err := i.iter.Next()
314-
if err != nil {
315-
return nil, nil, err
316-
}
323+
for {
324+
if i.commits == nil {
325+
var err error
326+
i.repo, err = i.repos.Next()
327+
if err != nil {
328+
return nil, nil, err
329+
}
317330

318-
key, err := encodeIndexKey(packOffsetIndexKey{
319-
Repository: obj.RepositoryID,
320-
Packfile: obj.Packfile.String(),
321-
Offset: int64(obj.Offset),
322-
})
323-
if err != nil {
324-
return nil, nil, err
325-
}
331+
i.commits, err = i.repo.Repo.CommitObjects()
332+
if err != nil {
333+
return nil, nil, err
334+
}
326335

327-
commit, ok := obj.Object.(*object.Commit)
328-
if !ok {
329-
ErrInvalidObjectType.New(obj.Object, "*object.Commit")
330-
}
336+
r := i.pool.repositories[i.repo.ID]
337+
i.idx, err = newRepositoryIndex(r.path, r.kind)
338+
if err != nil {
339+
return nil, nil, err
340+
}
341+
}
331342

332-
row := commitToRow(obj.RepositoryID, commit)
333-
values, err := rowIndexValues(row, i.columns, CommitsSchema)
334-
if err != nil {
335-
return nil, nil, err
336-
}
343+
commit, err := i.commits.Next()
344+
if err != nil {
345+
if err == io.EOF {
346+
i.commits = nil
347+
continue
348+
}
349+
350+
return nil, nil, err
351+
}
337352

338-
return values, key, nil
353+
offset, packfile, err := i.idx.find(commit.Hash)
354+
if err != nil {
355+
return nil, nil, err
356+
}
357+
358+
var hash string
359+
if offset < 0 {
360+
hash = commit.Hash.String()
361+
}
362+
363+
key, err := encodeIndexKey(packOffsetIndexKey{
364+
Repository: i.repo.ID,
365+
Packfile: packfile.String(),
366+
Offset: offset,
367+
Hash: hash,
368+
})
369+
if err != nil {
370+
return nil, nil, err
371+
}
372+
373+
row := commitToRow(i.repo.ID, commit)
374+
values, err := rowIndexValues(row, i.columns, CommitsSchema)
375+
if err != nil {
376+
return nil, nil, err
377+
}
378+
379+
return values, key, nil
380+
}
339381
}
340382

341-
func (i *commitsKeyValueIter) Close() error { return i.iter.Close() }
383+
func (i *commitsKeyValueIter) Close() error { return i.repos.Close() }
342384

343385
type commitsIndexIter struct {
344386
index sql.IndexValueIter
345-
pool *RepositoryPool
346387
decoder *objectDecoder
347388
}
348389

390+
func newCommitsIndexIter(index sql.IndexValueIter, pool *RepositoryPool) *commitsIndexIter {
391+
return &commitsIndexIter{
392+
index: index,
393+
decoder: newObjectDecoder(pool),
394+
}
395+
}
396+
349397
func (i *commitsIndexIter) Next() (sql.Row, error) {
350-
data, err := i.index.Next()
398+
var err error
399+
var data []byte
400+
defer closeIndexOnError(&err, i.index)
401+
402+
data, err = i.index.Next()
351403
if err != nil {
352404
return nil, err
353405
}
354406

355407
var key packOffsetIndexKey
356-
if err := decodeIndexKey(data, &key); err != nil {
408+
if err = decodeIndexKey(data, &key); err != nil {
357409
return nil, err
358410
}
359411

360-
packfile := plumbing.NewHash(key.Packfile)
361-
if i.decoder == nil || !i.decoder.equals(key.Repository, packfile) {
362-
if i.decoder != nil {
363-
if err := i.decoder.Close(); err != nil {
364-
return nil, err
365-
}
366-
}
367-
368-
i.decoder, err = newObjectDecoder(i.pool.repositories[key.Repository], packfile)
369-
if err != nil {
370-
return nil, err
371-
}
372-
}
373-
374-
obj, err := i.decoder.get(key.Offset)
412+
obj, err := i.decoder.decode(
413+
key.Repository,
414+
plumbing.NewHash(key.Packfile),
415+
key.Offset,
416+
plumbing.NewHash(key.Hash),
417+
)
375418
if err != nil {
376419
return nil, err
377420
}

0 commit comments

Comments
 (0)