diff --git a/.golangci.yaml b/.golangci.yaml index 55c4c9f..532888c 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -22,6 +22,10 @@ linters: - makezero - unparam - revive + - goimports + - goconst + - unparam + - staticcheck disable: - wsl # Enable presets. @@ -50,4 +54,6 @@ linters: - name: var-naming arguments: allowed-names: ^[a-z]+([A-Z][a-z0-9]+)*$ # Regex for camelCase - forbidden-names: ^[_a-z]+(_[a-z0-9]+)*$ # Regex for snake_case \ No newline at end of file + forbidden-names: ^[_a-z]+(_[a-z0-9]+)*$ # Regex for snake_case + goconst: + min-occurrences: 3 \ No newline at end of file diff --git a/kfile/fileMgr.go b/kfile/fileMgr.go index 5411a57..a3727dc 100644 --- a/kfile/fileMgr.go +++ b/kfile/fileMgr.go @@ -91,25 +91,42 @@ func NewMetaData(created_at time.Time) FileMetadata { } } func (fm *FileMgr) PreallocateFile(blk *BlockId, size int64) error { + if err := fm.validatePreallocationParams(blk, size); err != nil { + return err + } + + filename := blk.GetFileName() + if err := fm.validatePermissions(); err != nil { + return err + } + + return fm.performPreallocation(filename, size) +} + +func (fm *FileMgr) validatePreallocationParams(blk *BlockId, size int64) error { if size%int64(fm.blocksize) != 0 { return fmt.Errorf("size must be multiple of blocksize %d", fm.blocksize) } - filename := blk.GetFileName() - if filename == "" { + if blk.GetFileName() == "" { return fmt.Errorf("invalid filename") } + return nil +} +func (fm *FileMgr) validatePermissions() error { dirStat, err := os.Stat(fm.dbDirectory) if err != nil { return fmt.Errorf("failed to stat directory: %v", err) } - dirMode := dirStat.Mode() - isDirWritable := dirMode&0200 != 0 - if !isDirWritable { + + if dirStat.Mode()&0200 == 0 { return fmt.Errorf("directory is not writable") } + return nil +} +func (fm *FileMgr) performPreallocation(filename string, size int64) error { f, err := fm.getFile(filename) if err != nil { return fmt.Errorf("failed to get file for preallocation: %v", err) @@ -120,24 +137,19 @@ func (fm *FileMgr) PreallocateFile(blk *BlockId, size int64) error { return fmt.Errorf("failed to stat file: %v", err) } - mode := stat.Mode() - isWritable := mode&0200 != 0 - if !isWritable { + if stat.Mode()&0200 == 0 { return fmt.Errorf("file is not writable") } - currentSize := stat.Size() - if currentSize >= size { + if stat.Size() >= size { return nil } - err = f.Truncate(size) - if err != nil { + if err := f.Truncate(size); err != nil { return fmt.Errorf("failed to preallocate sparse file: %v", err) } - err = f.Sync() - if err != nil { + if err := f.Sync(); err != nil { return fmt.Errorf("failed to sync preallocated file: %v", err) } diff --git a/kfile/slotted_page.go b/kfile/slotted_page.go index 2cba4ed..5d33a24 100644 --- a/kfile/slotted_page.go +++ b/kfile/slotted_page.go @@ -71,7 +71,7 @@ func (sp *SlottedPage) InsertCell(cell *Cell) error { } // Find insertion point in slot array (binary search by key) - insertPos := sp.findSlotPosition(cell.key) + insertPos := sp.FindSlotPosition(cell.key) // Insert new offset into slot array sp.slots = append(sp.slots, 0) @@ -91,7 +91,7 @@ func (sp *SlottedPage) InsertCell(cell *Cell) error { } // Find position to insert new cell based on key -func (sp *SlottedPage) findSlotPosition(key []byte) int { +func (sp *SlottedPage) FindSlotPosition(key []byte) int { // Binary search through slots low, high := 0, len(sp.slots)-1 diff --git a/log/log_dir_test.go b/log/log_dir_test.go index 8ef05f3..731c2d9 100644 --- a/log/log_dir_test.go +++ b/log/log_dir_test.go @@ -2,7 +2,7 @@ package log import ( "bytes" - "encoding/binary" + "errors" "fmt" "os" "path/filepath" @@ -165,15 +165,27 @@ func TestAppendBoundary(t *testing.T) { // Append records to fill the block record := make([]byte, 50) // Record size - for i := 0; i < blockSize/len(record)-2; i++ { - logMgr.Append(record) + for i := 0; i < blockSize/len(record)-3; i++ { + lsn, cellKey, err := logMgr.Append(record) + if err != nil { + t.Errorf("Expected cell key and lsn got %s", err) + } + if lsn < 0 { + t.Errorf("Invalid lsn") + } + if cellKey == nil { + t.Errorf("Invalid CellKey") + } } - initialBlock := logMgr.currentBlock - logMgr.Append(record) + _, _, err = logMgr.Append(record) - if logMgr.currentBlock == initialBlock { - t.Errorf("Expected new block after boundary overflow, but block did not change") + var customErr *Error + if errors.As(err, &customErr) { + expected := "log operation Append failed: failed to insert cell page full" + if customErr.Error() != expected { + t.Errorf("Expected '%s' but got: '%s'", expected, customErr.Error()) + } } } @@ -228,28 +240,8 @@ func printLogRecords(t *testing.T, lm *LogMgr, msg string) { if err != nil { panic(err) } - //page := kfile.NewPageFromBytes(rec) - //s, err := page.GetStringWithOffset(0) - s, err := rec.GetValue() - var results string - if err != nil { - panic(err) - } - switch v := s.(type) { - case string: - fmt.Println("Value is a string:", v) - case []byte: - length := binary.BigEndian.Uint32(v[:4]) // Get the length from first 4 bytes - content := v[4 : 4+length] // Extract just the content bytes - results = string(content) - default: - fmt.Println("Unhandled type") - } - - //npos := utils.MaxLength(len(s)) - //val, _ := page.GetInt(npos) - val := string(rec.GetKey()) - t.Logf("[%s, %s]", results, val) + s := string(rec) + t.Logf("[%s]", s) } t.Log() } diff --git a/log/logmgr.go b/log/logmgr.go index 15a3630..218027f 100644 --- a/log/logmgr.go +++ b/log/logmgr.go @@ -143,42 +143,6 @@ func (lm *LogMgr) Append(logrec []byte) (int, []byte, error) { } lm.logBuffer.SetContents(logpage) - //logPage := lm.bm.Get(lm.currentBlock).GetContents() - - //boundary := logPage.GetFreeSpace() - // - //recsize := len(logrec) - //intBytes := int(unsafe.Sizeof(0)) - //bytesNeeded := recsize + intBytes - - //if (boundary - bytesNeeded) < intBytes { - // if err := lm.Flush(); err != nil { - // return 0, &Error{Op: "append", Err: fmt.Errorf("failed to flush: %v", err)} - // } - // - // if lm.currentBlock, _ = lm.appendNewBlock(); lm.currentBlock == nil { - // return 0, &Error{Op: "append", Err: fmt.Errorf("failed to append new block")} - // } - // //if err := lm.logBuffer.GetContents().SetInt(0, lm.fm.BlockSize()); err != nil { - // // return 0, &Error{Op: "Pin", Err: fmt.Errorf("failed to append initial block")} - // //} - // - // if err := lm.logBuffer.Flush(); err != nil { - // return 0, &Error{Op: "Pin", Err: fmt.Errorf("failed to append initial block")} - // } - // - // boundary = logPage.GetFreeSpace() - //} - - //recpos := boundary - bytesNeeded - //if err := logPage.SetBytes(recpos, logrec); err != nil { - // return 0, &Error{Op: "append", Err: fmt.Errorf("failed to set bytes: %v", err)} - //} - // - //if err := logPage.SetInt(0, recpos); err != nil { - // return 0, &Error{Op: "append", Err: fmt.Errorf("failed to update boundary: %v", err)} - //} - lm.latestLSN++ lm.logBuffer.MarkModified(-1, lm.latestLSN) return lm.latestLSN, cellKey, nil diff --git a/log/temp_test.go b/log/temp_test.go index 7336a45..9dbf68e 100644 --- a/log/temp_test.go +++ b/log/temp_test.go @@ -9,7 +9,6 @@ import ( "ultraSQL/buffer" "ultraSQL/kfile" "ultraSQL/utils" - "unsafe" ) func TestLogMgrAppend(t *testing.T) { @@ -49,11 +48,11 @@ func verifyMultipleRecordsInSingleBlock(t *testing.T, lm *LogMgr, blockSize int) t.Log("Testing appending multiple records within a single block...") // Append multiple small records - record1 := record("record2", 200) - record2 := record("record1", 100) + record1 := "record2" + record2 := "record1" - lsn1, _, err := lm.Append(record1) - lsn2, _, err := lm.Append(record2) + lsn1, _, err := lm.Append([]byte(record1)) + lsn2, _, err := lm.Append([]byte(record2)) if err != nil { t.Errorf("Error occured %s", err) } @@ -69,7 +68,7 @@ func verifyMultipleRecordsInSingleBlock(t *testing.T, lm *LogMgr, blockSize int) t.Errorf("Error occured %s", err) } records := readAllRecords(t, iter) - expected := []string{"record1, 100", "record2, 200"} + expected := []string{"record1", "record2"} compareRecords(t, records, expected) } @@ -80,8 +79,12 @@ func verifyRecordsAcrossBlocks(t *testing.T, lm *LogMgr, blockSize int) { // Each record is 1/5 of the block records := []string{} for i := 1; i <= 10; i++ { - record := record(fmt.Sprintf("record%d", i), i*10) - lm.Append(record) + str := string(rune(i)) + record := []byte(str) + _, _, err := lm.Append(record) + if err != nil { + return + } records = append(records, fmt.Sprintf("record%d, %d", i, i*10)) } @@ -94,23 +97,6 @@ func verifyRecordsAcrossBlocks(t *testing.T, lm *LogMgr, blockSize int) { compareRecords(t, readRecords, records) } -func record(s string, n int) []byte { - strOffset := utils.MaxLength(len(s)) - record := make([]byte, strOffset+int(unsafe.Sizeof(0))) // String + Integer - page := kfile.NewPageFromBytes(record) - - if err := page.SetString(0, s); err != nil { - panic(fmt.Sprintf("Failed to set string: %v", err)) - } - if err := page.SetInt(strOffset, n); err != nil { - panic(fmt.Sprintf("Failed to set int: %v", err)) - } - - // Log serialized record details - //fmt.Printf("Serialized record [%s, %d]: strOffset=%d, recordLen=%d\n", s, n, strOffset, len(record)) - return record -} - func readAllRecords(t *testing.T, iter utils.Iterator[[]byte]) []string { var records []string for iter.HasNext() { @@ -119,29 +105,10 @@ func readAllRecords(t *testing.T, iter utils.Iterator[[]byte]) []string { t.Fatalf("Error reading record: %v", err) } - //page := kfile.NewPageFromBytes(rec) - //s, err := page.GetString(0) - s, err := rec.GetValue() - if err != nil { - panic(err) - } - s, ok := s.(string) - if ok { - fmt.Println("Converted to string:", s) - } else { - fmt.Println("Value is not a string") - } - - //npos := utils.MaxLength(len(s)) // Ensure alignment - //n, err := page.GetInt(npos) - n := rec.GetKey() - if err != nil { - t.Fatalf("Error getting int: %v", err) - } + s := string(rec) - record := fmt.Sprintf("%s, %d", s, n) + record := fmt.Sprintf("%s", s) records = append(records, record) - //t.Logf("Deserialized record: [%s, %d] (npos=%d, recLen=%d)", s, n, npos, len(rec)) } return records } diff --git a/main.go b/main.go index 89a26dc..0695999 100644 --- a/main.go +++ b/main.go @@ -28,7 +28,7 @@ func main() { blk, err := fm.Append(Filename) checkError(err, "Failed to append block") fmt.Printf("Appended Block: %v\n", blk) - newPage := kfile.NewPage(blockSize) + newPage := kfile.NewSlottedPage(blockSize) err = newPage.SetInt(0, 42) checkError(err, "Failed to set int") @@ -46,7 +46,7 @@ func main() { err = fm.Write(blk, newPage) checkError(err, "Failed to write to block") - readPage := kfile.NewPage(blockSize) + readPage := kfile.NewSlottedPage(blockSize) err = fm.Read(blk, readPage) checkError(err, "Failed to read from block") diff --git a/utils/Iterator.go b/utils/Iterator.go index 8f55836..2485d26 100644 --- a/utils/Iterator.go +++ b/utils/Iterator.go @@ -1,8 +1,6 @@ package utils -import "ultraSQL/kfile" - type Iterator[T any] interface { HasNext() bool - Next() (*kfile.Cell, error) + Next() ([]byte, error) } diff --git a/utils/LogIterator.go b/utils/LogIterator.go index fa09e5e..005dcba 100644 --- a/utils/LogIterator.go +++ b/utils/LogIterator.go @@ -32,17 +32,27 @@ func (it *LogIterator) HasNext() bool { return it.currentPos >= 0 || it.blk.Number() > 0 } -func (it *LogIterator) Next() (*kfile.Cell, error) { +func (it *LogIterator) Next() ([]byte, error) { if it.currentPos < 0 { it.blk = kfile.NewBlockId(it.blk.GetFileName(), it.blk.Number()-1) if err := it.moveToBlock(it.blk); err != nil { return nil, err } } - rec, err := it.buff.GetContents().GetCellBySlot(it.currentPos) + cell, err := it.buff.GetContents().GetCellBySlot(it.currentPos) if err != nil { return nil, fmt.Errorf("error while getting bytes: %v", err) } + cellVal, err := cell.GetValue() + if err != nil { + return nil, fmt.Errorf("error while getting value: %v", err) + } + rec, ok := cellVal.([]byte) + + if !ok { + panic("value is not byte") + } + //recLen := string(rec) //npos := MaxLength(len(recLen)) //it.currentPos += int(unsafe.Sizeof(0)) + npos @@ -64,8 +74,6 @@ func (it *LogIterator) moveToBlock(blk *kfile.BlockId) error { return err } it.slots = it.buff.GetContents().GetAllSlots() - //it.boundary, _ = it.buff.GetContents().GetInt(0) - //it.currentPos = it.boundary it.currentPos = len(it.slots) - 1 return nil } diff --git a/utils/utils_test.go b/utils/utils_test.go index a49fdbf..8d1239f 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -1,6 +1,12 @@ package utils import ( + "bytes" + "encoding/binary" + "fmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "math/rand" "os" "path/filepath" _ "path/filepath" @@ -8,10 +14,6 @@ import ( "time" "ultraSQL/buffer" "ultraSQL/kfile" - "unsafe" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) // Helper function to create a temporary file manager @@ -56,125 +58,32 @@ func prepareLogBlock(t *testing.T, fm *kfile.FileMgr, filename string, records [ return blk } -func TestLogIterator_SingleBlockSingleRecord(t *testing.T) { - fm := createTempFileMgr(t) - filename := "test_single_record.log" - - testRecords := [][]byte{ - []byte("hello world"), +func randomCellKeyGenerator() []byte { + randNum := rand.Int31n(5) + buf := new(bytes.Buffer) + err := binary.Write(buf, binary.BigEndian, randNum) + if err != nil { + fmt.Println("Binary write failed:", err) + return nil } - - blk := prepareLogBlock(t, fm, filename, testRecords) - bm := buffer.NewBufferMgr(fm, 3) - iterator := NewLogIterator(fm, bm, blk) - - // Verify HasNext - assert.True(t, iterator.HasNext()) - - // Verify Next retrieves correct record - retrievedRec, _ := iterator.Next() - retrievedRec = retrievedRec[:len(retrievedRec)-int(unsafe.Sizeof(0))] - assert.Equal(t, testRecords[0], retrievedRec) - - // Verify no more records - assert.True(t, iterator.HasNext()) + return buf.Bytes() } - -//func TestLogIterator_MultipleRecordsSameBlock(t *testing.T) { -// fm := createTempFileMgr(t) -// filename := "test_multiple_records.log" -// -// testRecords := [][]byte{ -// []byte("first record"), -// []byte("second record"), -// []byte("third record"), -// } -// -// blk := prepareLogBlock(t, fm, filename, testRecords) -// -// iterator := NewLogIterator(fm, blk) -// -// // Verify records are retrieved in reverse order -// for i := len(testRecords) - 1; i >= 0; i-- { -// assert.True(t, iterator.HasNext()) -// retrievedRec, _ := iterator.Next() -// assert.Equal(t, testRecords[i], retrievedRec) -// } -// -// assert.False(t, iterator.HasNext()) -//} -// -//func TestLogIterator_MultipleBlocks(t *testing.T) { -// fm := createTempFileMgr(t) -// filename := "test_multiple_blocks.log" -// -// // Prepare first block -// firstBlockRecords := [][]byte{ -// []byte("first block first record"), -// []byte("first block second record"), -// } -// firstBlk := prepareLogBlock(t, fm, filename, firstBlockRecords) -// -// // Prepare second block -// secondBlockRecords := [][]byte{ -// []byte("second block first record"), -// []byte("second block second record"), -// } -// secondBlk := kfile.NewBlockId(filename, 1) -// secondPage := kfile.NewPage(fm.BlockSize()) -// secondPage.SetInt(0, int(4)) -// -// currentPos := 4 -// for _, rec := range secondBlockRecords { -// err := secondPage.SetBytes(currentPos, rec) -// require.NoError(t, err) -// -// currentPos += int(unsafe.Sizeof(int(0))) + len(rec) -// secondPage.SetInt(0, int(currentPos)) -// } -// -// err := fm.Write(secondBlk, secondPage) -// require.NoError(t, err) -// -// // Create iterator starting from first block -// iterator := NewLogIterator(fm, firstBlk) -// -// // Expected records in reverse order -// expectedRecords := append(firstBlockRecords, secondBlockRecords...) -// -// // Verify records retrieval across blocks -// for i := len(expectedRecords) - 1; i >= 0; i-- { -// assert.True(t, iterator.HasNext()) -// retrievedRec, _ := iterator.Next() -// assert.Equal(t, expectedRecords[i], retrievedRec) -// } -// -// assert.False(t, iterator.HasNext()) -//} - func TestLogIterator_EmptyIterator(t *testing.T) { fm := createTempFileMgr(t) filename := "test_empty.log" // Create an empty block - blk := kfile.NewBlockId(filename, 0) - page := kfile.NewPage(fm.BlockSize()) - page.SetInt(0, int(4)) // Set boundary to start of page + blk := kfile.NewBlockId(filename, 0) + page := kfile.NewSlottedPage(fm.BlockSize()) err := fm.Write(blk, page) require.NoError(t, err) bm := buffer.NewBufferMgr(fm, 3) iterator := NewLogIterator(fm, bm, blk) - assert.True(t, iterator.HasNext()) + assert.False(t, iterator.HasNext()) } -//func TestLogIterator_NilFileMgr(t *testing.T) { -// iterator := NewLogIterator(nil, nil) -// -// assert.False(t, iterator.HasNext()) -//} - func setupTestFileMgr(t *testing.T) (*kfile.FileMgr, string) { tempDir := filepath.Join(os.TempDir(), "simpledb_test_"+time.Now().Format("20060102150405")) blockSize := 400 @@ -194,113 +103,21 @@ func TestMoveToBlock(t *testing.T) { filename := "test.db" block := kfile.NewBlockId(filename, 0) - page := kfile.NewPage(fm.BlockSize()) - - // Write a boundary value to the page - page.SetInt(0, 200) // Arbitrary boundary position - err := fm.Write(block, page) - if err != nil { - t.Fatalf("Failed to write block: %v", err) - } + page := kfile.NewSlottedPage(fm.BlockSize()) + cellKey := randomCellKeyGenerator() + cell := kfile.NewKVCell(cellKey) + err := cell.SetValue(200) + require.NoError(t, err) + err = page.InsertCell(cell) + buff := buffer.NewBuffer(fm) + buff.SetContents(page) bm := buffer.NewBufferMgr(fm, 3) // Initialize LogIterator and move to block - iter := &LogIterator{fm: fm, blk: block, bm: bm} - iter.moveToBlock(block) + iter := NewLogIterator(fm, bm, block) + err = iter.moveToBlock(block) + require.NoError(t, err) - if iter.boundary != 200 { - t.Errorf("Expected boundary to be 200, got %d", iter.boundary) - } - if iter.currentPos != 200 { - t.Errorf("Expected currentPos to be 200, got %d", iter.currentPos) + if iter.currentPos == 0 { + t.Errorf("Expected currentPos to be 0, got %d", iter.currentPos) } } - -//func TestHasNext(t *testing.T) { -// fm, tempDir := setupTestFileMgr(t) -// defer func() { -// fm.Close() -// os.RemoveAll(tempDir) -// }() -// -// filename := "test.db" -// block := kfile.NewBlockId(filename, 0) -// page := kfile.NewPage(fm.BlockSize()) -// -// // Write a boundary value to simulate data -// page.SetInt(0, 200) -// err := fm.Write(block, page) -// if err != nil { -// t.Fatalf("Failed to write block: %v", err) -// } -// -// // Initialize LogIterator -// iter := NewLogIterator(fm, block) -// -// // Check `HasNext` when there is data -// if !iter.HasNext() { -// t.Errorf("Expected HasNext to be true, got false") -// } -// -// // Simulate reaching the end of the block -// iter.currentPos = fm.BlockSize() -// if iter.HasNext() { -// t.Errorf("Expected HasNext to be false after reaching end of block, got true") -// } -//} - -//func TestMultipleBlocks(t *testing.T) { -// fm, tempDir := setupTestFileMgr(t) -// defer func() { -// fm.Close() -// os.RemoveAll(tempDir) -// }() -// -// filename := "test.db" -// block1 := kfile.NewBlockId(filename, 1) -// block2 := kfile.NewBlockId(filename, 0) -// -// // Write records to two blocks -// record1 := []byte("record in block 1") -// record2 := []byte("record in block 0") -// rec3 := make([]byte, fm.BlockSize()) -// rec4 := make([]byte, fm.BlockSize()) -// copy(rec3, record1) -// copy(rec4, record2) -// page1 := kfile.NewPageFromBytes(rec3) -// page2 := kfile.NewPageFromBytes(rec4) -// page1.SetBytes(200, record1) -// page1.SetInt(0, 200) // Set boundary for block 1 -// page2.SetBytes(225, record2) -// page2.SetInt(0, 225) // Set boundary for block 0 -// -// err := fm.Write(block1, page1) -// if err != nil { -// t.Fatalf("Failed to write block1: %v", err) -// } -// -// err = fm.Write(block2, page2) -// if err != nil { -// t.Fatalf("Failed to write block2: %v", err) -// } -// -// // Initialize LogIterator with the most recent block (block1) -// iter := NewLogIterator(fm, block1) -// -// // Retrieve record from block 1 -// if !iter.HasNext() { -// t.Fatalf("Expected HasNext to be true for block1") -// } -// rec1, _ := iter.Next() -// if string(rec1) != string(record1) { -// t.Errorf("Expected record '%s', got '%s'", string(record1), string(rec1)) -// } -// -// // Retrieve record from block 0 after transitioning -// if !iter.HasNext() { -// t.Fatalf("Expected HasNext to be true for block0") -// } -// rec2, _ := iter.Next() -// if string(rec2) != string(record2) { -// t.Errorf("Expected record '%s', got '%s'", string(record2), string(rec2)) -// } -//}