From 016e108c1c4bb15dd8bf09e2ead321b6283c1a1b Mon Sep 17 00:00:00 2001 From: Daniel Gruber Date: Sun, 3 Nov 2024 20:07:00 +0100 Subject: [PATCH 1/2] EH: Watch qacct file; improved types --- cmd/simulator/go.sum | 4 +- examples/testexample/go.mod | 5 +- examples/testexample/go.sum | 2 + examples/testexample/testexample.go | 78 ++++++++++-- go.mod | 1 + go.sum | 2 + pkg/qacct/v9.0/file.go | 90 ++++++++++++++ pkg/qacct/v9.0/file_test.go | 66 ++++++++++ pkg/qacct/v9.0/parse.go | 83 ++++++++----- pkg/qacct/v9.0/parse_test.go | 50 ++++++-- pkg/qacct/v9.0/types.go | 101 ++++++++------- pkg/qstat/v9.0/parser.go | 5 + pkg/qstat/v9.0/parser_test.go | 184 ++++++++++++++++++++++++++++ pkg/qstat/v9.0/qstat_impl.go | 8 ++ 14 files changed, 579 insertions(+), 100 deletions(-) create mode 100644 pkg/qacct/v9.0/file.go create mode 100644 pkg/qacct/v9.0/file_test.go create mode 100644 pkg/qstat/v9.0/parser_test.go diff --git a/cmd/simulator/go.sum b/cmd/simulator/go.sum index e741c24..d04547d 100644 --- a/cmd/simulator/go.sum +++ b/cmd/simulator/go.sum @@ -22,8 +22,8 @@ golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= diff --git a/examples/testexample/go.mod b/examples/testexample/go.mod index cfd68d1..202c47e 100644 --- a/examples/testexample/go.mod +++ b/examples/testexample/go.mod @@ -10,4 +10,7 @@ require ( google.golang.org/protobuf v1.35.1 ) -require go.uber.org/multierr v1.10.0 // indirect +require ( + github.com/goccy/go-json v0.10.3 // indirect + go.uber.org/multierr v1.10.0 // indirect +) diff --git a/examples/testexample/go.sum b/examples/testexample/go.sum index 64ed0e3..dff5629 100644 --- a/examples/testexample/go.sum +++ b/examples/testexample/go.sum @@ -4,6 +4,8 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= diff --git a/examples/testexample/testexample.go b/examples/testexample/testexample.go index 6dd8fcf..a6cb8f5 100644 --- a/examples/testexample/testexample.go +++ b/examples/testexample/testexample.go @@ -17,21 +17,34 @@ import ( var qacctClient qacct.QAcct var qstatClient qstat.QStat +var newlyFinishedJobs <-chan qacct.JobDetail + var log *zap.Logger func init() { var err error log, _ = zap.NewProduction() + + qstatClient, err = qstat.NewCommandLineQstat(qstat.CommandLineQStatConfig{}) + if err != nil { + log.Fatal("Failed to initialize qstat client", zap.String("error", + err.Error())) + } + qacctClient, err = qacct.NewCommandLineQAcct(qacct.CommandLineQAcctConfig{}) if err != nil { log.Fatal("Failed to initialize qacct client", zap.String("error", err.Error())) } - qstatClient, err = qstat.NewCommandLineQstat(qstat.CommandLineQStatConfig{}) + + // watch for newly finished jobs + newlyFinishedJobs, err = qacct.WatchFile(context.Background(), + qacct.GetDefaultQacctFile(), 1024) if err != nil { - log.Fatal("Failed to initialize qstat client", zap.String("error", - err.Error())) + log.Fatal("Failed to initialize job watcher", + zap.String("error", err.Error())) } + } func main() { @@ -48,7 +61,7 @@ func run(ctx context.Context) { log.Info("Context cancelled, stopping ClusterScheduler") return default: - finishedJobs, err := GetFinishedJobs() + finishedJobs, err := GetFinishedJobsWithWatcher() if err != nil { log.Error("Error getting finished jobs", zap.String("error", err.Error())) @@ -107,10 +120,47 @@ type SimpleJob struct { MasterNode string `json:"master_node"` } +func GetFinishedJobsWithWatcher() ([]*SimpleJob, error) { + jobs := []*SimpleJob{} + + for { + // get next job or timeout after 0.1s of there is no new job + select { + case fjob := <-newlyFinishedJobs: + state := fmt.Sprintf("%d", fjob.ExitStatus) + if state == "0" { + state = "done" + } else { + state = "failed" + } + simpleJob := SimpleJob{ + // ignore job arrays for now + JobId: fmt.Sprintf("%d", fjob.JobNumber), + Cluster: fjob.QName, + JobName: fjob.JobName, + Partition: fjob.GrantedPE, + Account: fjob.Account, + User: fjob.Owner, + State: state, + ExitCode: fmt.Sprintf("%d", fjob.ExitStatus), + Submit: parseTimestampInt64(fjob.SubmitTime), + Start: parseTimestampInt64(fjob.StartTime), + End: parseTimestampInt64(fjob.EndTime), + MasterNode: fjob.HostName, + } + jobs = append(jobs, &simpleJob) + case <-time.After(100 * time.Millisecond): + return jobs, nil + } + } + return jobs, nil +} + func GetFinishedJobs() ([]*SimpleJob, error) { // Use qacct NativeSpecification to get finished jobs qacctOutput, err := qacctClient.NativeSpecification([]string{"-j", "*"}) if err != nil { + // no job are command failed return nil, fmt.Errorf("error running qacct command: %v", err) } @@ -137,9 +187,9 @@ func GetFinishedJobs() ([]*SimpleJob, error) { User: job.Owner, State: state, ExitCode: fmt.Sprintf("%d", job.ExitStatus), - Submit: parseTimestamp(job.QSubTime), - Start: parseTimestamp(job.StartTime), - End: parseTimestamp(job.EndTime), + Submit: parseTimestampInt64(job.SubmitTime), + Start: parseTimestampInt64(job.StartTime), + End: parseTimestampInt64(job.EndTime), MasterNode: job.HostName, } } @@ -150,7 +200,8 @@ func GetRunningJobs() ([]*SimpleJob, error) { qstatOverview, err := qstatClient.NativeSpecification([]string{"-g", "t"}) if err != nil { - return nil, fmt.Errorf("error running qstat command: %v", err) + // no jobs running + return nil, nil } jobsByTask, err := qstat.ParseGroupByTask(qstatOverview) if err != nil { @@ -193,7 +244,8 @@ func GetRunningJobs() ([]*SimpleJob, error) { // get running jobs qstatOutput, err := qstatClient.NativeSpecification([]string{"-j", "*"}) if err != nil { - return nil, fmt.Errorf("error running qstat command: %v", err) + // no jobs running; qstat -j * found 0 jobs (TODO) + return nil, nil } jobs, err := qstat.ParseSchedulerJobInfo(qstatOutput) @@ -242,6 +294,14 @@ func SendJobs(ctx context.Context, jobs []*SimpleJob) (int, error) { return len(jobs), nil } +func parseTimestampInt64(ts int64) *timestamppb.Timestamp { + // ts is 6 digits behind the second (microseconds) + sec := ts / 1e6 + nsec := (ts - sec*1e6) * 1e3 + t := time.Unix(sec, nsec) + return timestamppb.New(t) +} + // 2024-10-24 09:49:59.911136 func parseTimestamp(s string) *timestamppb.Timestamp { loc, err := time.LoadLocation("Local") diff --git a/go.mod b/go.mod index 2ed5b84..4165b8c 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/hpc-gridware/go-clusterscheduler go 1.22.4 require ( + github.com/goccy/go-json v0.10.3 github.com/onsi/ginkgo/v2 v2.19.1 github.com/onsi/gomega v1.34.1 go.opentelemetry.io/contrib/bridges/otelslog v0.5.0 diff --git a/go.sum b/go.sum index daec351..1f70016 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= diff --git a/pkg/qacct/v9.0/file.go b/pkg/qacct/v9.0/file.go new file mode 100644 index 0000000..4a3512a --- /dev/null +++ b/pkg/qacct/v9.0/file.go @@ -0,0 +1,90 @@ +package qacct + +import ( + "bufio" + "context" + "fmt" + "io" + "log" + "os" + "path/filepath" + "time" + + "github.com/goccy/go-json" +) + +// DefaultQacctFile returns the path to the default accounting file based +// on the SGE_ROOT and SGE_CELL environment variables. +func GetDefaultQacctFile() string { + sgeRoot := os.Getenv("SGE_ROOT") + sgeCell := os.Getenv("SGE_CELL") + return filepath.Join(sgeRoot, sgeCell, "common", "accounting.jsonl") +} + +// WatchFile returns a channel that emits all JobDetail objects from the accounting +// file. It continues to emit JobDetail objects as new lines are added to the file. +// The channel is buffered with the given buffer size. +func WatchFile(ctx context.Context, path string, bufferSize int) (<-chan JobDetail, error) { + if path == "" { + path = GetDefaultQacctFile() + } + + file, err := os.OpenFile(path, os.O_RDONLY, 0) + if err != nil { + return nil, fmt.Errorf("failed to open file: %v", err) + } + + jobDetailsChan := make(chan JobDetail, bufferSize) + + // offset points to the last processed line + var offset int64 = 0 + + go func() { + defer file.Close() + defer close(jobDetailsChan) + + scanner := bufio.NewScanner(file) + + for { + if _, err := file.Seek(offset, io.SeekStart); err != nil { + log.Printf("failed to seek to file end: %v", err) + return + } + + for scanner.Scan() { + var job JobDetail + line := scanner.Text() + // TODO parsing can be done in parallel + err := json.Unmarshal([]byte(line), &job) + if err != nil { + log.Printf("failed to unmarshal line: %v", err) + continue + } + jobDetailsChan <- job + } + + if err := scanner.Err(); err != nil { + log.Printf("JSONL parsing error: %v", err) + return + } + + // store processed offset + offset, err = file.Seek(0, io.SeekCurrent) + if err != nil { + log.Printf("failed to get current offset: %v", err) + return + } + + // wait a little before re-scanning for new data and reset scanner + select { + case <-ctx.Done(): + return + default: + <-time.After(1 * time.Second) + scanner = bufio.NewScanner(file) + } + } + }() + + return jobDetailsChan, nil +} diff --git a/pkg/qacct/v9.0/file_test.go b/pkg/qacct/v9.0/file_test.go new file mode 100644 index 0000000..d223804 --- /dev/null +++ b/pkg/qacct/v9.0/file_test.go @@ -0,0 +1,66 @@ +package qacct_test + +import ( + "context" + "fmt" + "log" + "slices" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + qacct "github.com/hpc-gridware/go-clusterscheduler/pkg/qacct/v9.0" + qsub "github.com/hpc-gridware/go-clusterscheduler/pkg/qsub/v9.0" +) + +var _ = Describe("File", func() { + + Context("WatchFile", func() { + + It("returns an error when the file does not exist", func() { + _, err := qacct.WatchFile(context.Background(), + "nonexistentfile.txt", 10) + Expect(err).To(HaveOccurred()) + }) + + It("returns a channel that emits JobDetail objects for 10 jobs", func() { + + qs, err := qsub.NewCommandLineQSub(qsub.CommandLineQSubConfig{}) + Expect(err).NotTo(HaveOccurred()) + + jobIDs := make([]int, 10) + for i := 0; i < 10; i++ { + jobID, _, err := qs.Submit(context.Background(), qsub.JobOptions{ + Command: "echo", + CommandArgs: []string{fmt.Sprintf("job %d", i+1)}, + Binary: qsub.ToPtr(true), + }) + Expect(err).NotTo(HaveOccurred()) + log.Printf("jobID: %d", jobID) + jobIDs[i] = int(jobID) + } + + jobDetailsChan, err := qacct.WatchFile(context.Background(), + qacct.GetDefaultQacctFile(), 0) + Expect(err).NotTo(HaveOccurred()) + Expect(jobDetailsChan).NotTo(BeNil()) + + receivedJobs := make(map[int]bool) + Eventually(func() bool { + select { + case jd := <-jobDetailsChan: + log.Printf("job: %+v", jd.JobNumber) + // check if jobID is in the jobIDs list + if slices.Contains(jobIDs, int(jd.JobNumber)) { + Expect(jd.SubmitCommandLine).To(ContainSubstring("echo 'job")) + Expect(jd.JobUsage.Usage.Memory).To(BeNumerically(">=", 0)) + receivedJobs[int(jd.JobNumber)] = true + } + default: + return len(receivedJobs) == 10 + } + return false + }, "10s").Should(BeTrue()) + }) + }) +}) diff --git a/pkg/qacct/v9.0/parse.go b/pkg/qacct/v9.0/parse.go index d14c8c5..673d3bd 100644 --- a/pkg/qacct/v9.0/parse.go +++ b/pkg/qacct/v9.0/parse.go @@ -21,8 +21,10 @@ package qacct import ( "bufio" + "encoding/json" "strconv" "strings" + "time" ) func ParseQacctJobOutputWithScanner(scanner *bufio.Scanner) ([]JobDetail, error) { @@ -73,13 +75,13 @@ func ParseQacctJobOutputWithScanner(scanner *bufio.Scanner) ([]JobDetail, error) case "priority": job.Priority = parseInt64(value) case "qsub_time": - job.QSubTime = value + job.SubmitTime = parseTime(value) case "submit_cmd_line": job.SubmitCommandLine = value case "start_time": - job.StartTime = value + job.StartTime = parseTime(value) case "end_time": - job.EndTime = value + job.EndTime = parseTime(value) case "granted_pe": job.GrantedPE = value case "slots": @@ -89,55 +91,53 @@ func ParseQacctJobOutputWithScanner(scanner *bufio.Scanner) ([]JobDetail, error) case "exit_status": job.ExitStatus = parseInt64(value) case "ru_wallclock": - job.RuWallClock = parseFloat(value) + job.JobUsage.RUsage.RuWallclock = parseInt64(value) case "ru_utime": - job.RuUTime = parseFloat(value) + job.JobUsage.RUsage.RuUtime = parseFloat(value) case "ru_stime": - job.RuSTime = parseFloat(value) + job.JobUsage.RUsage.RuStime = parseFloat(value) case "ru_maxrss": - job.RuMaxRSS = parseInt64(value) + job.JobUsage.RUsage.RuMaxrss = parseInt64(value) case "ru_ixrss": - job.RuIXRSS = parseInt64(value) + job.JobUsage.RUsage.RuIxrss = parseInt64(value) case "ru_ismrss": - job.RuISMRSS = parseInt64(value) + job.JobUsage.RUsage.RuIsmrss = parseInt64(value) case "ru_idrss": - job.RuIDRSS = parseInt64(value) + job.JobUsage.RUsage.RuIdrss = parseInt64(value) case "ru_isrss": - job.RuISRss = parseInt64(value) + job.JobUsage.RUsage.RuIsrss = parseInt64(value) case "ru_minflt": - job.RuMinFlt = parseInt64(value) + job.JobUsage.RUsage.RuMinflt = parseInt64(value) case "ru_majflt": - job.RuMajFlt = parseInt64(value) + job.JobUsage.RUsage.RuMajflt = parseInt64(value) case "ru_nswap": - job.RuNSwap = parseInt64(value) + job.JobUsage.RUsage.RuNswap = parseInt64(value) case "ru_inblock": - job.RuInBlock = parseInt64(value) + job.JobUsage.RUsage.RuInblock = parseInt64(value) case "ru_oublock": - job.RuOuBlock = parseInt64(value) + job.JobUsage.RUsage.RuOublock = parseInt64(value) case "ru_msgsnd": - job.RuMsgSend = parseInt64(value) + job.JobUsage.RUsage.RuMsgsnd = parseInt64(value) case "ru_msgrcv": - job.RuMsgRcv = parseInt64(value) + job.JobUsage.RUsage.RuMsgrcv = parseInt64(value) case "ru_nsignals": - job.RuNSignals = parseInt64(value) + job.JobUsage.RUsage.RuNsignals = parseInt64(value) case "ru_nvcsw": - job.RuNVCSw = parseInt64(value) - case "ru_nivcsw": - job.RuNiVCSw = parseInt64(value) + job.JobUsage.RUsage.RuNvcsw = parseInt64(value) case "wallclock": - job.WallClock = parseFloat(value) + job.JobUsage.Usage.WallClock = parseFloat(value) case "cpu": - job.CPU = parseFloat(value) + job.JobUsage.Usage.CPU = parseFloat(value) case "mem": - job.Memory = parseFloat(value) + job.JobUsage.Usage.Memory = parseFloat(value) case "io": - job.IO = parseFloat(value) + job.JobUsage.Usage.IO = parseFloat(value) case "iow": - job.IOWait = parseFloat(value) + job.JobUsage.Usage.IOWait = parseFloat(value) case "maxvmem": - job.MaxVMem = parseInt64(value) + job.JobUsage.Usage.MaxVMem = parseFloat(value) case "maxrss": - job.MaxRSS = parseInt64(value) + job.JobUsage.Usage.MaxRSS = parseFloat(value) case "arid": job.ArID = value } @@ -161,6 +161,22 @@ func ParseQAcctJobOutput(output string) ([]JobDetail, error) { return jobs, nil } +/* +qsub_time 2024-09-27 07:41:44.421951 +submit_cmd_line qsub -b y -t 1-100:2 sleep 0 +start_time 2024-09-27 07:42:07.265733 +end_time 2024-09-27 07:42:08.796845 +*/ +func parseTime(value string) int64 { + // fix layout to match the output: "2024-09-27 07:42:08.796845" + layout := "2006-01-02 15:04:05.999999" // Correct layout for the given examples + t, err := time.Parse(layout, value) + if err != nil { + return 0 + } + return t.UnixNano() / 1000 +} + func parseInt(value string) int { i, _ := strconv.Atoi(value) return i @@ -175,3 +191,12 @@ func parseFloat(value string) float64 { f, _ := strconv.ParseFloat(value, 64) return f } + +func ParseAccountingJSONLine(line string) (JobDetail, error) { + var job JobDetail + err := json.Unmarshal([]byte(line), &job) + if err != nil { + return JobDetail{}, err + } + return job, nil +} diff --git a/pkg/qacct/v9.0/parse_test.go b/pkg/qacct/v9.0/parse_test.go index 2f2dec9..a78e5bc 100644 --- a/pkg/qacct/v9.0/parse_test.go +++ b/pkg/qacct/v9.0/parse_test.go @@ -148,26 +148,26 @@ Total System Usage Expect(job1.JobNumber).To(Equal(int64(8))) Expect(job1.TaskID).To(Equal(int64(97))) Expect(job1.Account).To(Equal("sge")) - Expect(job1.QSubTime).To(Equal("2024-09-27 07:41:44.421951")) - Expect(job1.StartTime).To(Equal("2024-09-27 07:42:07.272221")) - Expect(job1.EndTime).To(Equal("2024-09-27 07:42:08.801865")) + Expect(job1.SubmitTime).To(Equal(int64(1727422904421951))) + Expect(job1.StartTime).To(Equal(int64(1727422927272221))) + Expect(job1.EndTime).To(Equal(int64(1727422928801865))) Expect(job1.Failed).To(Equal(int64(0))) Expect(job1.ExitStatus).To(Equal(int64(0))) - Expect(job1.RuWallClock).To(Equal(1.0)) - Expect(job1.RuUTime).To(Equal(0.492)) - Expect(job1.RuSTime).To(Equal(0.234)) - Expect(job1.RuMaxRSS).To(Equal(int64(10300))) - Expect(job1.MaxVMem).To(Equal(int64(21045248))) - Expect(job1.MaxRSS).To(Equal(int64(10547200))) + Expect(job1.JobUsage.Usage.WallClock).To(Equal(3.487)) + Expect(job1.JobUsage.RUsage.RuUtime).To(Equal(0.492)) + Expect(job1.JobUsage.RUsage.RuStime).To(Equal(0.234)) + Expect(job1.JobUsage.RUsage.RuMaxrss).To(Equal(int64(10300))) + Expect(job1.JobUsage.Usage.MaxVMem).To(Equal(float64(21045248))) + Expect(job1.JobUsage.Usage.MaxRSS).To(Equal(float64(10547200))) job2 := jobs[1] Expect(job2.QName).To(Equal("all.q")) Expect(job2.HostName).To(Equal("master")) Expect(job2.JobNumber).To(Equal(int64(8))) Expect(job2.TaskID).To(Equal(int64(99))) - Expect(job2.QSubTime).To(Equal("2024-09-27 07:41:44.421951")) - Expect(job2.StartTime).To(Equal("2024-09-27 07:42:07.265733")) - Expect(job2.EndTime).To(Equal("2024-09-27 07:42:08.796845")) + Expect(job2.SubmitTime).To(Equal(int64(1727422904421951))) + Expect(job2.StartTime).To(Equal(int64(1727422927265733))) + Expect(job2.EndTime).To(Equal(int64(1727422928796845))) }) It("should handle empty input", func() { @@ -188,4 +188,30 @@ Total System Usage }) }) + + Context("Raw JSON", func() { + + sampleOutput := `{"job_number":10,"task_number":1,"start_time":1730532913429415,"end_time":1730532913979016,"owner":"root","group":"root","account":"sge","qname":"all.q","hostname":"master","department":"defaultdepartment","slots":1,"job_name":"echo","priority":0,"submission_time":1730532912874519,"submit_cmd_line":"qsub -b y -terse echo 'job 1'","category":"","failed":0,"exit_status":0,"usage":{"rusage":{"ru_wallclock":0,"ru_utime":0.355821,"ru_stime":0.161309,"ru_maxrss":10284,"ru_ixrss":0,"ru_ismrss":0,"ru_idrss":0,"ru_isrss":0,"ru_minflt":504,"ru_majflt":0,"ru_nswap":0,"ru_inblock":0,"ru_oublock":11,"ru_msgsnd":0,"ru_msgrcv":0,"ru_nsignals":0,"ru_nvcsw":248,"ru_nivcsw":14},"usage":{"wallclock":2.022342,"cpu":0.51713,"mem":0.0043125152587890625,"io":0.000008341856300830841,"iow":0.0,"maxvmem":21049344.0,"maxrss":10530816.0}}}` + + It("should parse raw JSON correctly", func() { + job, err := qacct.ParseAccountingJSONLine(sampleOutput) + Expect(err).To(BeNil()) + Expect(job).NotTo(BeNil()) + Expect(job.JobNumber).To(Equal(int64(10))) + Expect(job.TaskID).To(Equal(int64(1))) + Expect(job.StartTime).To(Equal(int64(1730532913429415))) + Expect(job.EndTime).To(Equal(int64(1730532913979016))) + Expect(job.SubmitTime).To(Equal(int64(1730532912874519))) + Expect(job.SubmitCommandLine).To(Equal("qsub -b y -terse echo 'job 1'")) + Expect(job.JobName).To(Equal("echo")) + Expect(job.Account).To(Equal("sge")) + Expect(job.Priority).To(Equal(int64(0))) + Expect(job.Failed).To(Equal(int64(0))) + Expect(job.ExitStatus).To(Equal(int64(0))) + Expect(job.JobUsage.RUsage.RuWallclock).To(Equal(int64(0))) + Expect(job.JobUsage.RUsage.RuUtime).To(Equal(float64(0.355821))) + Expect(job.JobUsage.RUsage.RuStime).To(Equal(float64(0.161309))) + Expect(job.JobUsage.RUsage.RuMaxrss).To(Equal(int64(10284))) + }) + }) }) diff --git a/pkg/qacct/v9.0/types.go b/pkg/qacct/v9.0/types.go index 1556331..0260783 100644 --- a/pkg/qacct/v9.0/types.go +++ b/pkg/qacct/v9.0/types.go @@ -85,53 +85,58 @@ type TaskUsage struct { JobDetail JobDetail } +// sampleOutput := `{"job_number":10,"task_number":1,"start_time":1730532913429415,"end_time":1730532913979016,"owner":"root","group":"root","account":"sge","qname":"all.q","hostname":"master","department":"defaultdepartment","slots":1,"job_name":"echo","priority":0,"submission_time":1730532912874519,"submit_cmd_line":"qsub -b y -terse echo 'job 1'","category":"","failed":0,"exit_status":0,"usage":{"rusage":{"ru_wallclock":0,"ru_utime":0.355821,"ru_stime":0.161309,"ru_maxrss":10284,"ru_ixrss":0,"ru_ismrss":0,"ru_idrss":0,"ru_isrss":0,"ru_minflt":504,"ru_majflt":0,"ru_nswap":0,"ru_inblock":0,"ru_oublock":11,"ru_msgsnd":0,"ru_msgrcv":0,"ru_nsignals":0,"ru_nvcsw":248,"ru_nivcsw":14},"usage":{"wallclock":2.022342,"cpu":0.51713,"mem":0.0043125152587890625,"io":0.000008341856300830841,"iow":0.0,"maxvmem":21049344.0,"maxrss":10530816.0}}}` + type JobDetail struct { - QName string `json:"qname"` - HostName string `json:"hostname"` - Group string `json:"group"` - Owner string `json:"owner"` - Project string `json:"project"` - Department string `json:"department"` - JobName string `json:"jobname"` - JobNumber int64 `json:"jobnumber"` - TaskID int64 `json:"taskid"` - PETaskID string `json:"pe_taskid"` - Account string `json:"account"` - Priority int64 `json:"priority"` - QSubTime string `json:"qsub_time"` - SubmitCommandLine string `json:"submit_command_line"` - StartTime string `json:"start_time"` - EndTime string `json:"end_time"` - GrantedPE string `json:"granted_pe"` - Slots int64 `json:"slots"` - Failed int64 `json:"failed"` - ExitStatus int64 `json:"exit_status"` - RuWallClock float64 `json:"ru_wallclock"` - RuUTime float64 `json:"ru_utime"` - RuSTime float64 `json:"ru_stime"` - RuMaxRSS int64 `json:"ru_maxrss"` - RuIXRSS int64 `json:"ru_ixrss"` - RuISMRSS int64 `json:"ru_ismrss"` - RuIDRSS int64 `json:"ru_idrss"` - RuISRss int64 `json:"ru_isrss"` - RuMinFlt int64 `json:"ru_minflt"` - RuMajFlt int64 `json:"ru_majflt"` - RuNSwap int64 `json:"ru_nswap"` - RuInBlock int64 `json:"ru_inblock"` - RuOuBlock int64 `json:"ru_oublock"` - RuMsgSend int64 `json:"ru_msgsnd"` - RuMsgRcv int64 `json:"ru_msgrcv"` - RuNSignals int64 `json:"ru_nsignals"` - RuNVCSw int64 `json:"ru_nvcsw"` - RuNiVCSw int64 `json:"ru_nivcsw"` - WallClock float64 `json:"wallclock"` - CPU float64 `json:"cpu"` - Memory float64 `json:"mem"` - IO float64 `json:"io"` - IOWait float64 `json:"iow"` - MaxVMem int64 `json:"maxvmem"` - MaxRSS int64 `json:"maxrss"` - ArID string `json:"arid"` + QName string `json:"qname"` + HostName string `json:"hostname"` + Group string `json:"group"` + Owner string `json:"owner"` + Project string `json:"project"` + Department string `json:"department"` + JobName string `json:"job_name"` + JobNumber int64 `json:"job_number"` + TaskID int64 `json:"task_number"` + PETaskID string `json:"pe_taskid"` + Account string `json:"account"` + Priority int64 `json:"priority"` + SubmitTime int64 `json:"submission_time"` + SubmitCommandLine string `json:"submit_cmd_line"` + StartTime int64 `json:"start_time"` + EndTime int64 `json:"end_time"` + GrantedPE string `json:"granted_pe"` + Slots int64 `json:"slots"` + Failed int64 `json:"failed"` + ExitStatus int64 `json:"exit_status"` + ArID string `json:"arid"` + JobUsage JobUsage `json:"usage"` +} + +type JobUsage struct { + Usage Usage `json:"usage"` + RUsage RUsage `json:"rusage"` +} + +// RUsage represents the resource usage data structure. +type RUsage struct { + RuWallclock int64 `json:"ru_wallclock"` + RuUtime float64 `json:"ru_utime"` + RuStime float64 `json:"ru_stime"` + RuMaxrss int64 `json:"ru_maxrss"` + RuIxrss int64 `json:"ru_ixrss"` + RuIsmrss int64 `json:"ru_ismrss"` + RuIdrss int64 `json:"ru_idrss"` + RuIsrss int64 `json:"ru_isrss"` + RuMinflt int64 `json:"ru_minflt"` + RuMajflt int64 `json:"ru_majflt"` + RuNswap int64 `json:"ru_nswap"` + RuInblock int64 `json:"ru_inblock"` + RuOublock int64 `json:"ru_oublock"` + RuMsgsnd int64 `json:"ru_msgsnd"` + RuMsgrcv int64 `json:"ru_msgrcv"` + RuNsignals int64 `json:"ru_nsignals"` + RuNvcsw int64 `json:"ru_nvcsw"` + RuNivcsw int64 `json:"ru_nivcsw"` } type PeUsage struct { @@ -144,7 +149,9 @@ type Usage struct { UserTime float64 `json:"utime"` SystemTime float64 `json:"stime"` CPU float64 `json:"cpu"` - Memory float64 `json:"memory"` + Memory float64 `json:"mem"` IO float64 `json:"io"` IOWait float64 `json:"iow"` + MaxVMem float64 `json:"maxvmem"` + MaxRSS float64 `json:"maxrss"` } diff --git a/pkg/qstat/v9.0/parser.go b/pkg/qstat/v9.0/parser.go index 4ec3207..cdb30bc 100644 --- a/pkg/qstat/v9.0/parser.go +++ b/pkg/qstat/v9.0/parser.go @@ -60,6 +60,11 @@ func ParseGroupByTask(input string) ([]ParallelJobTask, error) { func parseFixedWidthJobs(input string) ([]ParallelJobTask, error) { var tasks []ParallelJobTask + input = strings.TrimSpace(input) + if input == "" { + return tasks, nil + } + // Correct column positions based on your description columnPositions := []struct { start int diff --git a/pkg/qstat/v9.0/parser_test.go b/pkg/qstat/v9.0/parser_test.go new file mode 100644 index 0000000..093ce7a --- /dev/null +++ b/pkg/qstat/v9.0/parser_test.go @@ -0,0 +1,184 @@ +package qstat_test + +import ( + qstat "github.com/hpc-gridware/go-clusterscheduler/pkg/qstat/v9.0" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Parser", func() { + + Context("ParseGroupByTask", func() { + + It("should parse the output of qstat -g t", func() { + input := `job-ID prior name user state submit/start at queue master ja-task-ID +------------------------------------------------------------------------------------------------------------------ + 14 0.50500 sleep root r 2024-10-28 07:21:41 all.q@master MASTER + 15 0.50500 sleep root r 2024-10-28 07:26:14 all.q@master MASTER 1 + 15 0.50500 sleep root r 2024-10-28 07:26:14 all.q@master MASTER 3 + 15 0.50500 sleep root r 2024-10-28 07:26:14 all.q@master MASTER 5 + 17 0.60500 sleep root qw 2024-10-28 07:27:50 + 12 0.50500 sleep root qw 2024-10-28 07:17:34 + 15 0.50500 sleep root qw 2024-10-28 07:26:14 7-99:2` + jobs, err := qstat.ParseGroupByTask(input) + Expect(err).NotTo(HaveOccurred()) + Expect(len(jobs)).To(Equal(7)) + Expect(jobs[0].JobID).To(Equal(14)) + Expect(jobs[0].Name).To(Equal("sleep")) + Expect(jobs[0].User).To(Equal("root")) + Expect(jobs[0].State).To(Equal("r")) + Expect(jobs[0].SubmitStartAt).To(Equal("2024-10-28 07:21:41")) + Expect(jobs[0].Queue).To(Equal("all.q@master")) + Expect(jobs[0].Master).To(Equal("MASTER")) + Expect(jobs[0].TaskID).To(Equal("")) + Expect(jobs[1].JobID).To(Equal(15)) + Expect(jobs[1].Name).To(Equal("sleep")) + Expect(jobs[1].User).To(Equal("root")) + Expect(jobs[1].State).To(Equal("r")) + Expect(jobs[1].SubmitStartAt).To(Equal("2024-10-28 07:26:14")) + Expect(jobs[1].TaskID).To(Equal("1")) + Expect(jobs[2].JobID).To(Equal(15)) + Expect(jobs[2].Name).To(Equal("sleep")) + Expect(jobs[2].User).To(Equal("root")) + Expect(jobs[2].State).To(Equal("r")) + Expect(jobs[2].SubmitStartAt).To(Equal("2024-10-28 07:26:14")) + Expect(jobs[2].TaskID).To(Equal("3")) + Expect(jobs[3].JobID).To(Equal(15)) + Expect(jobs[3].Name).To(Equal("sleep")) + Expect(jobs[3].User).To(Equal("root")) + Expect(jobs[3].State).To(Equal("r")) + Expect(jobs[3].SubmitStartAt).To(Equal("2024-10-28 07:26:14")) + Expect(jobs[3].TaskID).To(Equal("5")) + Expect(jobs[4].JobID).To(Equal(17)) + Expect(jobs[4].Name).To(Equal("sleep")) + Expect(jobs[4].User).To(Equal("root")) + Expect(jobs[4].State).To(Equal("qw")) + Expect(jobs[4].SubmitStartAt).To(Equal("2024-10-28 07:27:50")) + Expect(jobs[4].TaskID).To(Equal("")) + Expect(jobs[5].JobID).To(Equal(12)) + Expect(jobs[5].Name).To(Equal("sleep")) + Expect(jobs[5].User).To(Equal("root")) + Expect(jobs[5].State).To(Equal("qw")) + Expect(jobs[5].SubmitStartAt).To(Equal("2024-10-28 07:17:34")) + Expect(jobs[5].TaskID).To(Equal("")) + }) + + It("should parse the output of qstat -g t", func() { + + output := `job-ID prior name user state submit/start at queue master ja-task-ID +------------------------------------------------------------------------------------------------------------------ + 19 0.60500 sleep root r 2024-10-28 08:34:25 all.q@master SLAVE + all.q@master SLAVE + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@master MASTER 1 + all.q@master SLAVE 1 + all.q@master SLAVE 1 + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@master SLAVE 2 + all.q@master SLAVE 2 + 19 0.60500 sleep root r 2024-10-28 08:34:25 all.q@sim1 SLAVE + all.q@sim1 SLAVE + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim1 SLAVE 1 + all.q@sim1 SLAVE 1 + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim1 SLAVE 2 + all.q@sim1 SLAVE 2 + all.q@sim1 SLAVE 2 + 19 0.60500 sleep root r 2024-10-28 08:34:25 all.q@sim10 SLAVE + all.q@sim10 SLAVE + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim10 SLAVE 1 + all.q@sim10 SLAVE 1 + all.q@sim10 SLAVE 1 + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim10 SLAVE 2 + all.q@sim10 SLAVE 2 + 19 0.60500 sleep root r 2024-10-28 08:34:25 all.q@sim11 SLAVE + all.q@sim11 SLAVE + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim11 SLAVE 1 + all.q@sim11 SLAVE 1 + all.q@sim11 SLAVE 1 + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim11 SLAVE 2 + all.q@sim11 SLAVE 2 + 18 0.50500 sleep root r 2024-10-28 08:33:57 all.q@sim12 MASTER + 19 0.60500 sleep root r 2024-10-28 08:34:25 all.q@sim12 SLAVE + all.q@sim12 SLAVE + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim12 SLAVE 1 + all.q@sim12 SLAVE 1 + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim12 SLAVE 2 + all.q@sim12 SLAVE 2 + 19 0.60500 sleep root r 2024-10-28 08:34:25 all.q@sim2 SLAVE + all.q@sim2 SLAVE + all.q@sim2 SLAVE + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim2 SLAVE 1 + all.q@sim2 SLAVE 1 + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim2 SLAVE 2 + all.q@sim2 SLAVE 2 + 19 0.60500 sleep root r 2024-10-28 08:34:25 all.q@sim3 MASTER + all.q@sim3 SLAVE + all.q@sim3 SLAVE + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim3 SLAVE 1 + all.q@sim3 SLAVE 1 + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim3 SLAVE 2 + all.q@sim3 SLAVE 2 + 19 0.60500 sleep root r 2024-10-28 08:34:25 all.q@sim4 SLAVE + all.q@sim4 SLAVE + all.q@sim4 SLAVE + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim4 SLAVE 1 + all.q@sim4 SLAVE 1 + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim4 SLAVE 2 + all.q@sim4 SLAVE 2 + 19 0.60500 sleep root r 2024-10-28 08:34:25 all.q@sim5 SLAVE + all.q@sim5 SLAVE + all.q@sim5 SLAVE + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim5 SLAVE 1 + all.q@sim5 SLAVE 1 + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim5 SLAVE 2 + all.q@sim5 SLAVE 2 + 19 0.60500 sleep root r 2024-10-28 08:34:25 all.q@sim6 SLAVE + all.q@sim6 SLAVE + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim6 SLAVE 1 + all.q@sim6 SLAVE 1 + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim6 SLAVE 2 + all.q@sim6 SLAVE 2 + all.q@sim6 SLAVE 2 + 19 0.60500 sleep root r 2024-10-28 08:34:25 all.q@sim7 SLAVE + all.q@sim7 SLAVE + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim7 SLAVE 1 + all.q@sim7 SLAVE 1 + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim7 MASTER 2 + all.q@sim7 SLAVE 2 + all.q@sim7 SLAVE 2 + 19 0.60500 sleep root r 2024-10-28 08:34:25 all.q@sim8 SLAVE + all.q@sim8 SLAVE + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim8 SLAVE 1 + all.q@sim8 SLAVE 1 + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim8 SLAVE 2 + all.q@sim8 SLAVE 2 + all.q@sim8 SLAVE 2 + 19 0.60500 sleep root r 2024-10-28 08:34:25 all.q@sim9 SLAVE + all.q@sim9 SLAVE + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim9 SLAVE 1 + all.q@sim9 SLAVE 1 + all.q@sim9 SLAVE 1 + 20 0.60500 sleep root r 2024-10-28 08:34:36 all.q@sim9 SLAVE 2 + all.q@sim9 SLAVE 2 + 12 0.50500 sleep root qw 2024-10-28 07:17:34` + + jobs, err := qstat.ParseGroupByTask(output) + Expect(err).NotTo(HaveOccurred()) + Expect(len(jobs)).To(Equal(41)) + + // last job + Expect(jobs[40].JobID).To(Equal(12)) + Expect(jobs[40].Name).To(Equal("sleep")) + Expect(jobs[40].User).To(Equal("root")) + Expect(jobs[40].State).To(Equal("qw")) + Expect(jobs[40].SubmitStartAt).To(Equal("2024-10-28 07:17:34")) + + // job before last + Expect(jobs[39].JobID).To(Equal(20)) + Expect(jobs[39].TaskID).To(Equal("2")) + Expect(jobs[39].Queue).To(Equal("all.q@sim9")) + Expect(jobs[39].Master).To(Equal("SLAVE")) + Expect(jobs[39].SubmitStartAt).To(Equal("2024-10-28 08:34:36")) + }) + + }) + +}) diff --git a/pkg/qstat/v9.0/qstat_impl.go b/pkg/qstat/v9.0/qstat_impl.go index 230e6c3..3a98eb5 100644 --- a/pkg/qstat/v9.0/qstat_impl.go +++ b/pkg/qstat/v9.0/qstat_impl.go @@ -128,6 +128,14 @@ func (q *QStatImpl) NativeSpecification(args []string) (string, error) { command := exec.Command(q.config.Executable, args...) out, err := command.Output() if err != nil { + // convert error in exit error + ee, ok := err.(*exec.ExitError) + if ok { + if !ee.Success() { + return "", fmt.Errorf("qstat command failed with exit code %d", ee.ExitCode()) + } + return "", nil + } return "", fmt.Errorf("failed to get output of qstat: %w", err) } return string(out), nil From c595e0aeced36849a6453f96918028a2ed7cca92 Mon Sep 17 00:00:00 2001 From: Daniel Gruber Date: Mon, 16 Dec 2024 15:52:17 +0100 Subject: [PATCH 2/2] EH: Add basic qhost functionality --- Dockerfile | 2 +- Makefile | 2 +- go.mod | 42 +-- go.sum | 97 ++--- pkg/adapter/rest.md | 2 +- pkg/{qacct/v9.0 => helper}/memory.go | 35 +- pkg/qhost/v9.0/parsers.go | 442 ++++++++++++++++++++++ pkg/qhost/v9.0/parsers_test.go | 422 +++++++++++++++++++++ pkg/qhost/v9.0/qhost.go | 27 ++ pkg/qhost/v9.0/qhost_impl.go | 99 +++++ pkg/qhost/v9.0/qhost_suite_test.go | 13 + pkg/qhost/v9.0/types.go | 87 +++++ pkg/qstat/v9.0/parser.go | 528 ++++++++++++++++++++++++++- pkg/qstat/v9.0/parser_test.go | 269 +++++++++++++- pkg/qstat/v9.0/qstat.go | 2 +- pkg/qstat/v9.0/qstat_impl.go | 16 +- pkg/qstat/v9.0/types.go | 21 +- 17 files changed, 1984 insertions(+), 122 deletions(-) rename pkg/{qacct/v9.0 => helper}/memory.go (77%) create mode 100644 pkg/qhost/v9.0/parsers.go create mode 100644 pkg/qhost/v9.0/parsers_test.go create mode 100644 pkg/qhost/v9.0/qhost.go create mode 100644 pkg/qhost/v9.0/qhost_impl.go create mode 100644 pkg/qhost/v9.0/qhost_suite_test.go create mode 100644 pkg/qhost/v9.0/types.go diff --git a/Dockerfile b/Dockerfile index dddc062..a03ad48 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ COPY entrypoint.sh /entrypoint.sh ARG GOLANG_VERSION=1.22.4 RUN apt-get update && \ - apt-get install -y wget git gcc make vim libhwloc-dev hwloc software-properties-common && \ + apt-get install -y curl wget git gcc make vim libhwloc-dev hwloc software-properties-common && \ add-apt-repository -y ppa:apptainer/ppa && \ apt-get update && \ apt-get install -y apptainer diff --git a/Makefile b/Makefile index 5256292..72be367 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ run: build simulate: @echo "Running the container in simulation mode using cluster.json" mkdir -p ./installation - docker run --rm -it -h master --privileged --cap-add SYS_ADMIN --name $(CONTAINER_NAME) -v ./installation:/opt/cs-install -v ./:/root/go/src/github.com/hpc-gridware/go-clusterscheduler $(IMAGE_NAME):$(IMAGE_TAG) /bin/bash -c "cd /root/go/src/github.com/hpc-gridware/go-clusterscheduler/cmd/simulator && go build . && ./simulator run ../../cluster.json && /bin/bash" + docker run --rm -it -h master --privileged --cap-add SYS_ADMIN -p 9464:9464 --name $(CONTAINER_NAME) -v ./installation:/opt/cs-install -v ./:/root/go/src/github.com/hpc-gridware/go-clusterscheduler $(IMAGE_NAME):$(IMAGE_TAG) /bin/bash -c "cd /root/go/src/github.com/hpc-gridware/go-clusterscheduler/cmd/simulator && go build . && ./simulator run ../../cluster.json && /bin/bash" #.PHONY: simulate #simulate: diff --git a/go.mod b/go.mod index 4165b8c..55ad560 100644 --- a/go.mod +++ b/go.mod @@ -1,20 +1,20 @@ module github.com/hpc-gridware/go-clusterscheduler -go 1.22.4 +go 1.23.1 require ( - github.com/goccy/go-json v0.10.3 - github.com/onsi/ginkgo/v2 v2.19.1 - github.com/onsi/gomega v1.34.1 - go.opentelemetry.io/contrib/bridges/otelslog v0.5.0 - go.opentelemetry.io/otel v1.30.0 - go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.6.0 - go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.30.0 - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.30.0 - go.opentelemetry.io/otel/log v0.6.0 - go.opentelemetry.io/otel/sdk v1.30.0 - go.opentelemetry.io/otel/sdk/log v0.6.0 - go.opentelemetry.io/otel/sdk/metric v1.30.0 + github.com/goccy/go-json v0.10.4 + github.com/onsi/ginkgo/v2 v2.22.0 + github.com/onsi/gomega v1.36.1 + go.opentelemetry.io/contrib/bridges/otelslog v0.8.0 + go.opentelemetry.io/otel v1.33.0 + go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.9.0 + go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.33.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.33.0 + go.opentelemetry.io/otel/log v0.9.0 + go.opentelemetry.io/otel/sdk v1.33.0 + go.opentelemetry.io/otel/sdk/log v0.9.0 + go.opentelemetry.io/otel/sdk/metric v1.33.0 ) require ( @@ -22,14 +22,14 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect github.com/google/uuid v1.6.0 // indirect - go.opentelemetry.io/otel/metric v1.30.0 // indirect - go.opentelemetry.io/otel/trace v1.30.0 // indirect - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/net v0.27.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.16.0 // indirect - golang.org/x/tools v0.23.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/metric v1.33.0 // indirect + go.opentelemetry.io/otel/trace v1.33.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.19.0 // indirect + golang.org/x/tools v0.26.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 1f70016..3862a1a 100644 --- a/go.sum +++ b/go.sum @@ -7,57 +7,64 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM= +github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= -github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/onsi/ginkgo/v2 v2.19.1 h1:QXgq3Z8Crl5EL1WBAC98A5sEBHARrAJNzAmMxzLcRF0= -github.com/onsi/ginkgo/v2 v2.19.1/go.mod h1:O3DtEWQkPa/F7fBMgmZQKKsluAy8pd3rEQdrjkPb9zA= -github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= -github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= +github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -go.opentelemetry.io/contrib/bridges/otelslog v0.5.0 h1:lU3F57OSLK5mQ1PDBVAfDDaKCPv37MrEbCfTzsF4bz0= -go.opentelemetry.io/contrib/bridges/otelslog v0.5.0/go.mod h1:I84u06zJFr8T5D73fslEUbnRBimVVSBhuVw8L8I92AU= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= -go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.6.0 h1:bZHOb8k/CwwSt0DgvgaoOhBXWNdWqFWaIsGTtg1H3KE= -go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.6.0/go.mod h1:XlV163j81kDdIt5b5BXCjdqVfqJFy/LJrHA697SorvQ= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.30.0 h1:IyFlqNsi8VT/nwYlLJfdM0y1gavxGpEvnf6FtVfZ6X4= -go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.30.0/go.mod h1:bxiX8eUeKoAEQmbq/ecUT8UqZwCjZW52yJrXJUSozsk= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.30.0 h1:kn1BudCgwtE7PxLqcZkErpD8GKqLZ6BSzeW9QihQJeM= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.30.0/go.mod h1:ljkUDtAMdleoi9tIG1R6dJUpVwDcYjw3J2Q6Q/SuiC0= -go.opentelemetry.io/otel/log v0.6.0 h1:nH66tr+dmEgW5y+F9LanGJUBYPrRgP4g2EkmPE3LeK8= -go.opentelemetry.io/otel/log v0.6.0/go.mod h1:KdySypjQHhP069JX0z/t26VHwa8vSwzgaKmXtIB3fJM= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= -go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= -go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= -go.opentelemetry.io/otel/sdk/log v0.6.0 h1:4J8BwXY4EeDE9Mowg+CyhWVBhTSLXVXodiXxS/+PGqI= -go.opentelemetry.io/otel/sdk/log v0.6.0/go.mod h1:L1DN8RMAduKkrwRAFDEX3E3TLOq46+XMGSbUfHU/+vE= -go.opentelemetry.io/otel/sdk/metric v1.30.0 h1:QJLT8Pe11jyHBHfSAgYH7kEmT24eX792jZO1bo4BXkM= -go.opentelemetry.io/otel/sdk/metric v1.30.0/go.mod h1:waS6P3YqFNzeP01kuo/MBBYqaoBJl7efRQHOaydhy1Y= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/bridges/otelslog v0.8.0 h1:G3sKsNueSdxuACINFxKrQeimAIst0A5ytA2YJH+3e1c= +go.opentelemetry.io/contrib/bridges/otelslog v0.8.0/go.mod h1:ptJm3wizguEPurZgarDAwOeX7O0iMR7l+QvIVenhYdE= +go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= +go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.9.0 h1:iI15wfQb5ZtAVTdS5WROxpYmw6Kjez3hT9SuzXhrgGQ= +go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.9.0/go.mod h1:yepwlNzVVxHWR5ugHIrll+euPQPq4pvysHTDr/daV9o= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.33.0 h1:FiOTYABOX4tdzi8A0+mtzcsTmi6WBOxk66u0f1Mj9Gs= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.33.0/go.mod h1:xyo5rS8DgzV0Jtsht+LCEMwyiDbjpsxBpWETwFRF0/4= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.33.0 h1:W5AWUn/IVe8RFb5pZx1Uh9Laf/4+Qmm4kJL5zPuvR+0= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.33.0/go.mod h1:mzKxJywMNBdEX8TSJais3NnsVZUaJ+bAy6UxPTng2vk= +go.opentelemetry.io/otel/log v0.9.0 h1:0OiWRefqJ2QszpCiqwGO0u9ajMPe17q6IscQvvp3czY= +go.opentelemetry.io/otel/log v0.9.0/go.mod h1:WPP4OJ+RBkQ416jrFCQFuFKtXKD6mOoYCQm6ykK8VaU= +go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= +go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= +go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= +go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= +go.opentelemetry.io/otel/sdk/log v0.9.0 h1:YPCi6W1Eg0vwT/XJWsv2/PaQ2nyAJYuF7UUjQSBe3bc= +go.opentelemetry.io/otel/sdk/log v0.9.0/go.mod h1:y0HdrOz7OkXQBuc2yjiqnEHc+CRKeVhRE3hx4RwTmV4= +go.opentelemetry.io/otel/sdk/metric v1.33.0 h1:Gs5VK9/WUJhNXZgn8MR6ITatvAmKeIuCtNbsP3JkNqU= +go.opentelemetry.io/otel/sdk/metric v1.33.0/go.mod h1:dL5ykHZmm1B1nVRk9dDjChwDmt81MjVp3gLkQRwKf/Q= +go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= +go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/adapter/rest.md b/pkg/adapter/rest.md index e15bf5e..90ad9f8 100644 --- a/pkg/adapter/rest.md +++ b/pkg/adapter/rest.md @@ -7,7 +7,7 @@ to execute `qconf` commands through http POST calls. ## Canonical Example of a qconf REST API -This is another example of exposing qconf trough a REST API. +This is another example of exposing qconf through a REST API. ### Cluster Configuration diff --git a/pkg/qacct/v9.0/memory.go b/pkg/helper/memory.go similarity index 77% rename from pkg/qacct/v9.0/memory.go rename to pkg/helper/memory.go index 449a586..22f1c05 100644 --- a/pkg/qacct/v9.0/memory.go +++ b/pkg/helper/memory.go @@ -17,13 +17,13 @@ ************************************************************************/ /*___INFO__MARK_END__*/ -package qacct +package helper import ( "errors" + "fmt" "strconv" "strings" - "unicode" ) // ParseMemoryFromString takes a string like "4.078G". @@ -36,32 +36,36 @@ import ( // 1000, K multiply by 1024, m multiply by 1000*1000, M multiply by 1024*1024, // g multiply by 1000*1000*1000 and G multiply by 1024*1024*1024. If no // multiplier is present, the value is just counted in bytes."" +// Example: "15.6G" -> 15.6 * 1024 * 1024 * 1024 func ParseMemoryFromString(m string) (int64, error) { if len(m) == 0 { return 0, errors.New("empty string") } - // Find position of the first non-digit character - var i int - for i = len(m) - 1; i >= 0; i-- { - if !unicode.IsDigit(rune(m[i])) && m[i] != '.' { - break - } + if m == "0" || m == "0.0" || m == "0.00" || m == "0.000" { + return 0, nil } - // Separate the number part and the multiplier part - numberStr := m[:i+1] - unit := m[i+1:] + // last character must be a multiplier + if !strings.HasSuffix(m, "k") && !strings.HasSuffix(m, "K") && + !strings.HasSuffix(m, "m") && !strings.HasSuffix(m, "M") && + !strings.HasSuffix(m, "g") && !strings.HasSuffix(m, "G") { + // no unit, return the number as is + return strconv.ParseInt(m, 10, 64) + } + + unit := m[len(m)-1] + numberStr := m[:len(m)-1] // Parse the number part number, err := strconv.ParseFloat(numberStr, 64) if err != nil { - return 0, err + return 0, fmt.Errorf("invalid number: %s: %v", numberStr, err) } // Determine the multiplier multiplier := int64(1) // Default is bytes if no unit - switch strings.ToUpper(unit) { + switch strings.ToUpper(string(unit)) { case "K": multiplier = 1024 case "M": @@ -80,10 +84,7 @@ func ParseMemoryFromString(m string) (int64, error) { return 0, errors.New("invalid unit") } - // Calculate the result - result := int64(number * float64(multiplier)) - - return result, nil + return int64(number * float64(multiplier)), nil } func MemoryToString(m int64) string { diff --git a/pkg/qhost/v9.0/parsers.go b/pkg/qhost/v9.0/parsers.go new file mode 100644 index 0000000..b0dff37 --- /dev/null +++ b/pkg/qhost/v9.0/parsers.go @@ -0,0 +1,442 @@ +/*___INFO__MARK_BEGIN__*/ +/************************************************************************* +* Copyright 2024 HPC-Gridware GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +************************************************************************/ +/*___INFO__MARK_END__*/ + +package qhost + +import ( + "fmt" + "strconv" + "strings" + + helper "github.com/hpc-gridware/go-clusterscheduler/pkg/helper" +) + +/* +Parses following output of qhost command: + +HOSTNAME ARCH NCPU NSOC NCOR NTHR LOAD MEMTOT MEMUSE SWAPTO SWAPUS +---------------------------------------------------------------------------------------------- +global - - - - - - - - - - +master lx-amd64 4 1 4 4 0.31 15.6G 422.9M 1.5G 0.0 +exec lx-amd64 4 1 4 4 0.31 15.6G 422.9M 1.5G 0.0 +... +*/ +func ParseHosts(out string) ([]Host, error) { + hosts := []Host{} + + for _, line := range strings.Split(out, "\n") { + if strings.HasPrefix(line, "HOSTNAME") { + continue + } + if strings.HasPrefix(line, "global") { + continue + } + if strings.HasPrefix(line, "---------------") { + continue + } + if strings.TrimSpace(line) == "" { + continue + } + // split line by whitespace + fields := strings.Fields(line) + if len(fields) < 11 { + return nil, fmt.Errorf("invalid line: %s", line) + } + + var err error + host := Host{} + host.Name = fields[0] + host.Arch = fields[1] + host.NCPU, err = strconv.Atoi(fields[2]) + if err != nil { + return nil, fmt.Errorf("invalid NCPU: %s", fields[2]) + } + + host.NSOC, err = strconv.Atoi(fields[3]) + if err != nil { + return nil, fmt.Errorf("invalid NSOC: %s", fields[3]) + } + host.NCOR, err = strconv.Atoi(fields[4]) + if err != nil { + return nil, fmt.Errorf("invalid NCOR: %s", fields[4]) + } + host.NTHR, err = strconv.Atoi(fields[5]) + if err != nil { + return nil, fmt.Errorf("invalid NTHR: %s", fields[5]) + } + host.LOAD, err = strconv.ParseFloat(fields[6], 64) + if err != nil { + return nil, fmt.Errorf("invalid LOAD: %s", fields[6]) + } + host.MEMTOT, err = helper.ParseMemoryFromString(fields[7]) + if err != nil { + return nil, fmt.Errorf("invalid MEMTOT: %s: %v", fields[7], err) + } + host.MEMUSE, err = helper.ParseMemoryFromString(fields[8]) + if err != nil { + return nil, fmt.Errorf("invalid MEMUSE: %s: %v", fields[8], err) + } + host.SWAPTO, err = helper.ParseMemoryFromString(fields[9]) + if err != nil { + return nil, fmt.Errorf("invalid SWAPTO: %s: %v", fields[9], err) + } + host.SWAPUS, err = helper.ParseMemoryFromString(fields[10]) + if err != nil { + return nil, fmt.Errorf("invalid SWAPUS: %s: %v", fields[10], err) + } + hosts = append(hosts, host) + } + return hosts, nil +} + +/* +HOSTNAME ARCH NCPU NSOC NCOR NTHR LOAD MEMTOT MEMUSE SWAPTO SWAPUS +---------------------------------------------------------------------------------------------- +global - - - - - - - - - - +master lx-amd64 4 1 4 4 0.34 15.6G 420.0M 1.5G 0.0 + + hl:arch=lx-amd64 + hl:num_proc=4.000000 + hl:mem_total=15.617G + hl:swap_total=1.500G + hl:virtual_total=17.117G + hl:load_avg=0.340000 + hl:load_short=0.550000 + hl:load_medium=0.340000 + hl:load_long=0.320000 + hl:mem_free=15.207G + hl:swap_free=1.500G + hl:virtual_free=16.707G + hl:mem_used=419.988M + hl:swap_used=0.000 + hl:virtual_used=419.988M + hl:cpu=0.000000 + hl:m_topology=SCCCC + hl:m_topology_inuse=SCCCC + hl:m_socket=1.000000 + hl:m_core=4.000000 + hl:m_thread=4.000000 + hl:np_load_avg=0.085000 + hl:np_load_short=0.137500 + hl:np_load_medium=0.085000 + hl:np_load_long=0.080000 + hc:NVIDIA_GPUS=2.000000 +*/ + +func ParseHostFullMetrics(out string) ([]HostFullMetrics, error) { + hosts := []HostFullMetrics{} + lines := strings.Split(out, "\n") + + var currentHost *HostFullMetrics = nil + + for i := 0; i < len(lines); i++ { + line := strings.TrimSpace(lines[i]) + if line == "" || strings.HasPrefix(line, "HOSTNAME") || + strings.HasPrefix(line, "global") || + strings.HasPrefix(line, "----") { + continue + } + + if strings.HasPrefix(lines[i], " ") { + // Attribute line or resource availability line + if currentHost == nil { + return nil, fmt.Errorf("attribute line encountered before any host line") + } + attributeLine := strings.TrimSpace(line) + err := parseAttributeLine(attributeLine, currentHost) + if err != nil { + return nil, fmt.Errorf("failed to parse attribute line: %v", err) + } + } else { + // New host line + // If currentHost is not nil, append it to hosts + if currentHost != nil { + hosts = append(hosts, *currentHost) + } + // Create new currentHost + currentHost = &HostFullMetrics{} + + // Parse host line + fields := strings.Fields(line) + if len(fields) < 10 { + return nil, fmt.Errorf("invalid host line: %s", line) + } + currentHost.Name = fields[0] + currentHost.Arch = fields[1] + + // Parse NCPU + ncpu, err := strconv.Atoi(fields[2]) + if err == nil { + currentHost.NumProc = float64(ncpu) + } + + // The LOAD field is at index 6 + loadAvg, err := strconv.ParseFloat(fields[6], 64) + if err == nil { + currentHost.LoadAvg = loadAvg + } + + // MEMTOT is at index 7 + memTotal, err := helper.ParseMemoryFromString(fields[7]) + if err == nil { + currentHost.MemTotal = memTotal + } + + // MEMUSE is at index 8 + memUsed, err := helper.ParseMemoryFromString(fields[8]) + if err == nil { + currentHost.MemUsed = memUsed + } + + // SWAPTO is at index 9 + swapTotal, err := helper.ParseMemoryFromString(fields[9]) + if err == nil { + currentHost.SwapTotal = swapTotal + } + + // SWAPUS is at index 10 + swapUsed, err := helper.ParseMemoryFromString(fields[10]) + if err == nil { + currentHost.SwapUsed = swapUsed + } + + // Initialize Resources map + currentHost.Resources = make(map[string]ResourceAvailability) + } + } + + // Append the last host + if currentHost != nil { + hosts = append(hosts, *currentHost) + } + + return hosts, nil +} + +func parseAttributeLine(line string, currentHost *HostFullMetrics) error { + // The line is expected to be in the format: + // [Availability][Source]:[resource_name]=[value] + // e.g., "hl:load_avg=0.600000" + + prefixAndRest := strings.SplitN(line, ":", 2) + if len(prefixAndRest) != 2 { + return fmt.Errorf("invalid attribute line format: %s", line) + } + prefix := prefixAndRest[0] + rest := prefixAndRest[1] + + // Extract availability and source + if len(prefix) != 2 { + return fmt.Errorf("invalid prefix length: %s", prefix) + } + availabilityLetter := prefix[0] + sourceLetter := prefix[1] + + // Now split rest into resource_name and value + attrParts := strings.SplitN(rest, "=", 2) + if len(attrParts) != 2 { + return fmt.Errorf("invalid attribute line, missing '=': %s", line) + } + resourceName := attrParts[0] + value := attrParts[1] + + availabilityMap := map[byte]string{ + 'g': "g", // cluster global + 'h': "h", // host total + 'q': "q", // queue total + } + sourceMap := map[byte]string{ + 'l': "l", // load value + 'L': "L", // load value after scaling + 'c': "c", // consumable resource + 'f': "F", // fixed availability + } + + resourceAvailabilityLimitedBy, ok := availabilityMap[availabilityLetter] + if !ok { + return fmt.Errorf("unknown availability letter: %c", availabilityLetter) + } + source, ok := sourceMap[sourceLetter] + if !ok { + return fmt.Errorf("unknown source letter: %c", sourceLetter) + } + + // Now process the resource_name and value + switch resourceName { + case "arch": + currentHost.Arch = value + case "num_proc": + numProc, err := strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("invalid num_proc value: %s", value) + } + currentHost.NumProc = numProc + case "mem_total": + memTotal, err := helper.ParseMemoryFromString(value) + if err != nil { + return fmt.Errorf("invalid mem_total value: %s", value) + } + currentHost.MemTotal = memTotal + case "swap_total": + swapTotal, err := helper.ParseMemoryFromString(value) + if err != nil { + return fmt.Errorf("invalid swap_total value: %s", value) + } + currentHost.SwapTotal = swapTotal + case "virtual_total": + virtualTotal, err := helper.ParseMemoryFromString(value) + if err != nil { + return fmt.Errorf("invalid virtual_total value: %s", value) + } + currentHost.VirtualTotal = virtualTotal + case "load_avg": + loadAvg, err := strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("invalid load_avg value: %s", value) + } + currentHost.LoadAvg = loadAvg + case "load_short": + loadShort, err := strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("invalid load_short value: %s", value) + } + currentHost.LoadShort = loadShort + case "load_medium": + loadMedium, err := strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("invalid load_medium value: %s", value) + } + currentHost.LoadMedium = loadMedium + case "load_long": + loadLong, err := strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("invalid load_long value: %s", value) + } + currentHost.LoadLong = loadLong + case "mem_free": + memFree, err := helper.ParseMemoryFromString(value) + if err != nil { + return fmt.Errorf("invalid mem_free value: %s", value) + } + currentHost.MemFree = memFree + case "swap_free": + swapFree, err := helper.ParseMemoryFromString(value) + if err != nil { + return fmt.Errorf("invalid swap_free value: %s", value) + } + currentHost.SwapFree = swapFree + case "virtual_free": + virtualFree, err := helper.ParseMemoryFromString(value) + if err != nil { + return fmt.Errorf("invalid virtual_free value: %s", value) + } + currentHost.VirtualFree = virtualFree + case "mem_used": + memUsed, err := helper.ParseMemoryFromString(value) + if err != nil { + return fmt.Errorf("invalid mem_used value: %s", value) + } + currentHost.MemUsed = memUsed + case "swap_used": + swapUsed, err := helper.ParseMemoryFromString(value) + if err != nil { + return fmt.Errorf("invalid swap_used value: %s", value) + } + currentHost.SwapUsed = swapUsed + case "virtual_used": + virtualUsed, err := helper.ParseMemoryFromString(value) + if err != nil { + return fmt.Errorf("invalid virtual_used value: %s", value) + } + currentHost.VirtualUsed = virtualUsed + case "cpu": + cpu, err := strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("invalid cpu value: %s", value) + } + currentHost.CPU = cpu + case "m_topology": + currentHost.Topology = value + case "m_topology_inuse": + currentHost.TopologyInuse = value + case "m_socket": + socket, err := strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("invalid m_socket value: %s", value) + } + currentHost.Socket = int64(socket) + case "m_core": + core, err := strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("invalid m_core value: %s", value) + } + currentHost.Core = int64(core) + case "m_thread": + thread, err := strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("invalid m_thread value: %s", value) + } + currentHost.Thread = int64(thread) + case "np_load_avg": + npLoadAvg, err := strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("invalid np_load_avg value: %s", value) + } + currentHost.NPLoadAvg = npLoadAvg + case "np_load_short": + npLoadShort, err := strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("invalid np_load_short value: %s", value) + } + currentHost.NPLoadShort = npLoadShort + case "np_load_medium": + npLoadMedium, err := strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("invalid np_load_medium value: %s", value) + } + currentHost.NPLoadMedium = npLoadMedium + case "np_load_long": + npLoadLong, err := strconv.ParseFloat(value, 64) + if err != nil { + return fmt.Errorf("invalid np_load_long value: %s", value) + } + currentHost.NPLoadLong = npLoadLong + default: + // Handle as a self-defined resource + if currentHost.Resources == nil { + currentHost.Resources = make(map[string]ResourceAvailability) + } + ra := ResourceAvailability{ + Name: resourceName, + StringValue: value, + ResourceAvailabilityLimitedBy: resourceAvailabilityLimitedBy, + Source: source, + FullString: line, + } + + // Try to parse the value as float + floatValue, err := strconv.ParseFloat(value, 64) + if err == nil { + ra.FloatValue = floatValue + } + + currentHost.Resources[resourceName] = ra + } + return nil +} diff --git a/pkg/qhost/v9.0/parsers_test.go b/pkg/qhost/v9.0/parsers_test.go new file mode 100644 index 0000000..afae316 --- /dev/null +++ b/pkg/qhost/v9.0/parsers_test.go @@ -0,0 +1,422 @@ +package qhost_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + qhost "github.com/hpc-gridware/go-clusterscheduler/pkg/qhost/v9.0" +) + +var _ = Describe("Parsers", func() { + + sample := ` +HOSTNAME ARCH NCPU NSOC NCOR NTHR LOAD MEMTOT MEMUSE SWAPTO SWAPUS +---------------------------------------------------------------------------------------------- +global - - - - - - - - - - +master lx-amd64 4 1 4 4 0.31 15.6G 422.9M 1.5G 0.0 +exec lx-amd64 4 1 4 4 0.31 15.6G 422.9M 1.5G 0.0 +` + Context("ParseQhostOutput", func() { + + It("should return error if output is invalid", func() { + hosts, err := qhost.ParseHosts(sample) + Expect(err).To(BeNil()) + Expect(hosts).To(HaveLen(2)) + Expect(hosts[0].Name).To(Equal("master")) + Expect(hosts[0].Arch).To(Equal("lx-amd64")) + Expect(hosts[0].NCPU).To(Equal(4)) + Expect(hosts[0].NSOC).To(Equal(1)) + Expect(hosts[0].NCOR).To(Equal(4)) + Expect(hosts[0].NTHR).To(Equal(4)) + Expect(hosts[0].LOAD).To(Equal(0.31)) + Expect(hosts[0].MEMTOT).To(Equal(int64(156 * 1024 * 1024 * 1024 / 10))) + Expect(hosts[0].MEMUSE).To(Equal(int64(4229 * 1024 * 1024 / 10))) + Expect(hosts[0].SWAPTO).To(Equal(int64(1.5 * 1024 * 1024 * 1024))) + Expect(hosts[0].SWAPUS).To(Equal(int64(0.0))) + Expect(hosts[1].Name).To(Equal("exec")) + Expect(hosts[1].Arch).To(Equal("lx-amd64")) + Expect(hosts[1].NCPU).To(Equal(4)) + Expect(hosts[1].NSOC).To(Equal(1)) + Expect(hosts[1].NCOR).To(Equal(4)) + Expect(hosts[1].NTHR).To(Equal(4)) + Expect(hosts[1].LOAD).To(Equal(0.31)) + Expect(hosts[1].MEMTOT).To(Equal(int64(156 * 1024 * 1024 * 1024 / 10))) + Expect(hosts[1].MEMUSE).To(Equal(int64(4229 * 1024 * 1024 / 10))) + Expect(hosts[1].SWAPTO).To(Equal(int64(1.5 * 1024 * 1024 * 1024))) + Expect(hosts[1].SWAPUS).To(Equal(int64(0.0))) + }) + + }) + + Context("ParseHostFullMetrics", func() { + + qhostFOutput1 := `HOSTNAME ARCH NCPU NSOC NCOR NTHR LOAD MEMTOT MEMUSE SWAPTO SWAPUS +---------------------------------------------------------------------------------------------- +global - - - - - - - - - - +master lx-amd64 4 1 4 4 0.60 15.6G 465.8M 1.5G 0.0 + hl:arch=lx-amd64 + hl:num_proc=4.000000 + hl:mem_total=15.617G + hl:swap_total=1.500G + hl:virtual_total=17.117G + hl:load_avg=0.600000 + hl:load_short=0.700000 + hl:load_medium=0.600000 + hl:load_long=0.440000 + hl:mem_free=15.162G + hl:swap_free=1.500G + hl:virtual_free=16.662G + hl:mem_used=465.824M + hl:swap_used=0.000 + hl:virtual_used=465.824M + hl:cpu=0.200000 + hl:m_topology=SCCCC + hl:m_topology_inuse=SCCCC + hl:m_socket=1.000000 + hl:m_core=4.000000 + hl:m_thread=4.000000 + hl:np_load_avg=0.150000 + hl:np_load_short=0.175000 + hl:np_load_medium=0.150000 + hl:np_load_long=0.110000 +sim1 lx-amd64 4 1 4 4 0.60 15.6G 465.8M 1.5G 0.0 + hl:load_avg=0.600000 + hl:load_short=0.700000 + hl:load_medium=0.600000 + hl:load_long=0.440000 + hl:arch=lx-amd64 + hl:num_proc=4.000000 + hl:mem_free=15.162G + hl:swap_free=1.500G + hl:virtual_free=16.662G + hl:mem_total=15.617G + hl:swap_total=1.500G + hl:virtual_total=17.117G + hl:mem_used=465.824M + hl:swap_used=0.000 + hl:virtual_used=465.824M + hl:cpu=0.200000 + hl:m_topology=SCCCC + hl:m_topology_inuse=SCCCC + hl:m_socket=1.000000 + hl:m_core=4.000000 + hl:m_thread=4.000000 + hl:np_load_avg=0.150000 + hl:np_load_short=0.175000 + hl:np_load_medium=0.150000 + hl:np_load_long=0.110000 + hf:load_report_host=master +sim10 lx-amd64 4 1 4 4 0.60 15.6G 465.8M 1.5G 0.0 + hl:load_avg=0.600000 + hl:load_short=0.700000 + hl:load_medium=0.600000 + hl:load_long=0.440000 + hl:arch=lx-amd64 + hl:num_proc=4.000000 + hl:mem_free=15.162G + hl:swap_free=1.500G + hl:virtual_free=16.662G + hl:mem_total=15.617G + hl:swap_total=1.500G + hl:virtual_total=17.117G + hl:mem_used=465.824M + hl:swap_used=0.000 + hl:virtual_used=465.824M + hl:cpu=0.200000 + hl:m_topology=SCCCC + hl:m_topology_inuse=SCCCC + hl:m_socket=1.000000 + hl:m_core=4.000000 + hl:m_thread=4.000000 + hl:np_load_avg=0.150000 + hl:np_load_short=0.175000 + hl:np_load_medium=0.150000 + hl:np_load_long=0.110000 + hf:load_report_host=master +sim11 lx-amd64 4 1 4 4 0.60 15.6G 465.8M 1.5G 0.0 + hl:load_avg=0.600000 + hl:load_short=0.700000 + hl:load_medium=0.600000 + hl:load_long=0.440000 + hl:arch=lx-amd64 + hl:num_proc=4.000000 + hl:mem_free=15.162G + hl:swap_free=1.500G + hl:virtual_free=16.662G + hl:mem_total=15.617G + hl:swap_total=1.500G + hl:virtual_total=17.117G + hl:mem_used=465.824M + hl:swap_used=0.000 + hl:virtual_used=465.824M + hl:cpu=0.200000 + hl:m_topology=SCCCC + hl:m_topology_inuse=SCCCC + hl:m_socket=1.000000 + hl:m_core=4.000000 + hl:m_thread=4.000000 + hl:np_load_avg=0.150000 + hl:np_load_short=0.175000 + hl:np_load_medium=0.150000 + hl:np_load_long=0.110000 + hf:load_report_host=master +sim12 lx-amd64 4 1 4 4 0.60 15.6G 465.8M 1.5G 0.0 + hl:load_avg=0.600000 + hl:load_short=0.700000 + hl:load_medium=0.600000 + hl:load_long=0.440000 + hl:arch=lx-amd64 + hl:num_proc=4.000000 + hl:mem_free=15.162G + hl:swap_free=1.500G + hl:virtual_free=16.662G + hl:mem_total=15.617G + hl:swap_total=1.500G + hl:virtual_total=17.117G + hl:mem_used=465.824M + hl:swap_used=0.000 + hl:virtual_used=465.824M + hl:cpu=0.200000 + hl:m_topology=SCCCC + hl:m_topology_inuse=SCCCC + hl:m_socket=1.000000 + hl:m_core=4.000000 + hl:m_thread=4.000000 + hl:np_load_avg=0.150000 + hl:np_load_short=0.175000 + hl:np_load_medium=0.150000 + hl:np_load_long=0.110000 + hf:load_report_host=master +sim2 lx-amd64 4 1 4 4 0.60 15.6G 465.8M 1.5G 0.0 + hl:load_avg=0.600000 + hl:load_short=0.700000 + hl:load_medium=0.600000 + hl:load_long=0.440000 + hl:arch=lx-amd64 + hl:num_proc=4.000000 + hl:mem_free=15.162G + hl:swap_free=1.500G + hl:virtual_free=16.662G + hl:mem_total=15.617G + hl:swap_total=1.500G + hl:virtual_total=17.117G + hl:mem_used=465.824M + hl:swap_used=0.000 + hl:virtual_used=465.824M + hl:cpu=0.200000 + hl:m_topology=SCCCC + hl:m_topology_inuse=SCCCC + hl:m_socket=1.000000 + hl:m_core=4.000000 + hl:m_thread=4.000000 + hl:np_load_avg=0.150000 + hl:np_load_short=0.175000 + hl:np_load_medium=0.150000 + hl:np_load_long=0.110000 + hf:load_report_host=master +sim3 lx-amd64 4 1 4 4 0.60 15.6G 465.8M 1.5G 0.0 + hl:load_avg=0.600000 + hl:load_short=0.700000 + hl:load_medium=0.600000 + hl:load_long=0.440000 + hl:arch=lx-amd64 + hl:num_proc=4.000000 + hl:mem_free=15.162G + hl:swap_free=1.500G + hl:virtual_free=16.662G + hl:mem_total=15.617G + hl:swap_total=1.500G + hl:virtual_total=17.117G + hl:mem_used=465.824M + hl:swap_used=0.000 + hl:virtual_used=465.824M + hl:cpu=0.200000 + hl:m_topology=SCCCC + hl:m_topology_inuse=SCCCC + hl:m_socket=1.000000 + hl:m_core=4.000000 + hl:m_thread=4.000000 + hl:np_load_avg=0.150000 + hl:np_load_short=0.175000 + hl:np_load_medium=0.150000 + hl:np_load_long=0.110000 + hf:load_report_host=master +sim4 lx-amd64 4 1 4 4 0.60 15.6G 465.8M 1.5G 0.0 + hl:load_avg=0.600000 + hl:load_short=0.700000 + hl:load_medium=0.600000 + hl:load_long=0.440000 + hl:arch=lx-amd64 + hl:num_proc=4.000000 + hl:mem_free=15.162G + hl:swap_free=1.500G + hl:virtual_free=16.662G + hl:mem_total=15.617G + hl:swap_total=1.500G + hl:virtual_total=17.117G + hl:mem_used=465.824M + hl:swap_used=0.000 + hl:virtual_used=465.824M + hl:cpu=0.200000 + hl:m_topology=SCCCC + hl:m_topology_inuse=SCCCC + hl:m_socket=1.000000 + hl:m_core=4.000000 + hl:m_thread=4.000000 + hl:np_load_avg=0.150000 + hl:np_load_short=0.175000 + hl:np_load_medium=0.150000 + hl:np_load_long=0.110000 + hf:load_report_host=master +sim5 lx-amd64 4 1 4 4 0.60 15.6G 465.8M 1.5G 0.0 + hl:load_avg=0.600000 + hl:load_short=0.700000 + hl:load_medium=0.600000 + hl:load_long=0.440000 + hl:arch=lx-amd64 + hl:num_proc=4.000000 + hl:mem_free=15.162G + hl:swap_free=1.500G + hl:virtual_free=16.662G + hl:mem_total=15.617G + hl:swap_total=1.500G + hl:virtual_total=17.117G + hl:mem_used=465.824M + hl:swap_used=0.000 + hl:virtual_used=465.824M + hl:cpu=0.200000 + hl:m_topology=SCCCC + hl:m_topology_inuse=SCCCC + hl:m_socket=1.000000 + hl:m_core=4.000000 + hl:m_thread=4.000000 + hl:np_load_avg=0.150000 + hl:np_load_short=0.175000 + hl:np_load_medium=0.150000 + hl:np_load_long=0.110000 + hf:load_report_host=master +sim6 lx-amd64 4 1 4 4 0.60 15.6G 465.8M 1.5G 0.0 + hl:load_avg=0.600000 + hl:load_short=0.700000 + hl:load_medium=0.600000 + hl:load_long=0.440000 + hl:arch=lx-amd64 + hl:num_proc=4.000000 + hl:mem_free=15.162G + hl:swap_free=1.500G + hl:virtual_free=16.662G + hl:mem_total=15.617G + hl:swap_total=1.500G + hl:virtual_total=17.117G + hl:mem_used=465.824M + hl:swap_used=0.000 + hl:virtual_used=465.824M + hl:cpu=0.200000 + hl:m_topology=SCCCC + hl:m_topology_inuse=SCCCC + hl:m_socket=1.000000 + hl:m_core=4.000000 + hl:m_thread=4.000000 + hl:np_load_avg=0.150000 + hl:np_load_short=0.175000 + hl:np_load_medium=0.150000 + hl:np_load_long=0.110000 + hf:load_report_host=master +sim7 lx-amd64 4 1 4 4 0.60 15.6G 465.8M 1.5G 0.0 + hl:load_avg=0.600000 + hl:load_short=0.700000 + hl:load_medium=0.600000 + hl:load_long=0.440000 + hl:arch=lx-amd64 + hl:num_proc=4.000000 + hl:mem_free=15.162G + hl:swap_free=1.500G + hl:virtual_free=16.662G + hl:mem_total=15.617G + hl:swap_total=1.500G + hl:virtual_total=17.117G + hl:mem_used=465.824M + hl:swap_used=0.000 + hl:virtual_used=465.824M + hl:cpu=0.200000 + hl:m_topology=SCCCC + hl:m_topology_inuse=SCCCC + hl:m_socket=1.000000 + hl:m_core=4.000000 + hl:m_thread=4.000000 + hl:np_load_avg=0.150000 + hl:np_load_short=0.175000 + hl:np_load_medium=0.150000 + hl:np_load_long=0.110000 + hf:load_report_host=master +sim8 lx-amd64 4 1 4 4 0.60 15.6G 465.8M 1.5G 0.0 + hl:load_avg=0.600000 + hl:load_short=0.700000 + hl:load_medium=0.600000 + hl:load_long=0.440000 + hl:arch=lx-amd64 + hl:num_proc=4.000000 + hl:mem_free=15.162G + hl:swap_free=1.500G + hl:virtual_free=16.662G + hl:mem_total=15.617G + hl:swap_total=1.500G + hl:virtual_total=17.117G + hl:mem_used=465.824M + hl:swap_used=0.000 + hl:virtual_used=465.824M + hl:cpu=0.200000 + hl:m_topology=SCCCC + hl:m_topology_inuse=SCCCC + hl:m_socket=1.000000 + hl:m_core=4.000000 + hl:m_thread=4.000000 + hl:np_load_avg=0.150000 + hl:np_load_short=0.175000 + hl:np_load_medium=0.150000 + hl:np_load_long=0.110000 + hf:load_report_host=master +sim9 lx-amd64 4 1 4 4 0.60 15.6G 465.8M 1.5G 0.0 + hl:load_avg=0.600000 + hl:load_short=0.700000 + hl:load_medium=0.600000 + hl:load_long=0.440000 + hl:arch=lx-amd64 + hl:num_proc=4.000000 + hl:mem_free=15.162G + hl:swap_free=1.500G + hl:virtual_free=16.662G + hl:mem_total=15.617G + hl:swap_total=1.500G + hl:virtual_total=17.117G + hl:mem_used=465.824M + hl:swap_used=0.000 + hl:virtual_used=465.824M + hl:cpu=0.200000 + hl:m_topology=SCCCC + hl:m_topology_inuse=SCCCC + hl:m_socket=1.000000 + hl:m_core=4.000000 + hl:m_thread=4.000000 + hl:np_load_avg=0.150000 + hl:np_load_short=0.175000 + hl:np_load_medium=0.150000 + hl:np_load_long=0.110000 + hf:load_report_host=master` + + It("should return error if output is invalid", func() { + hosts, err := qhost.ParseHostFullMetrics(sample) + Expect(err).To(BeNil()) + Expect(hosts).To(HaveLen(2)) + }) + + It("should parse host full metrics", func() { + hosts, err := qhost.ParseHostFullMetrics(qhostFOutput1) + Expect(err).To(BeNil()) + Expect(hosts).To(HaveLen(13)) + Expect(hosts[0].Name).To(Equal("master")) + Expect(hosts[12].Name).To(Equal("sim9")) + }) + }) + +}) diff --git a/pkg/qhost/v9.0/qhost.go b/pkg/qhost/v9.0/qhost.go new file mode 100644 index 0000000..263820c --- /dev/null +++ b/pkg/qhost/v9.0/qhost.go @@ -0,0 +1,27 @@ +/*___INFO__MARK_BEGIN__*/ +/************************************************************************* +* Copyright 2024 HPC-Gridware GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +************************************************************************/ +/*___INFO__MARK_END__*/ + +package qhost + +type QHost interface { + // GetHosts returns standard qhost output + GetHosts() ([]Host, error) + // GetHostFullMetrics returns qhost -F output + GetHostsFullMetrics() ([]HostFullMetrics, error) +} diff --git a/pkg/qhost/v9.0/qhost_impl.go b/pkg/qhost/v9.0/qhost_impl.go new file mode 100644 index 0000000..f5e1ee8 --- /dev/null +++ b/pkg/qhost/v9.0/qhost_impl.go @@ -0,0 +1,99 @@ +/*___INFO__MARK_BEGIN__*/ +/************************************************************************* +* Copyright 2024 HPC-Gridware GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +************************************************************************/ +/*___INFO__MARK_END__*/ + +package qhost + +import ( + "fmt" + "os/exec" +) + +type QHostImpl struct { + config CommandLineQHostConfig +} + +type CommandLineQHostConfig struct { + Executable string + DryRun bool +} + +func NewCommandLineQhost(config CommandLineQHostConfig) (*QHostImpl, error) { + if config.Executable == "" { + config.Executable = "qhost" + } + if config.DryRun == false { + // check if executable is reachable + _, err := exec.LookPath(config.Executable) + if err != nil { + return nil, fmt.Errorf("executable not found: %w", err) + } + } + return &QHostImpl{config: config}, nil +} + +// NativeSpecification returns the output of the qhost command for the given +// arguments. The arguments are passed to the qhost command as is. +// The output is returned as a string. +func (q *QHostImpl) NativeSpecification(args []string) (string, error) { + if q.config.DryRun { + return fmt.Sprintf("Dry run: qhost %v", args), nil + } + command := exec.Command(q.config.Executable, args...) + out, err := command.Output() + if err != nil { + // convert error in exit error + ee, ok := err.(*exec.ExitError) + if ok { + if !ee.Success() { + return "", fmt.Errorf("qhost command failed with exit code %d", + ee.ExitCode()) + } + return "", nil + } + return "", fmt.Errorf("failed to get output of qhost: %w", err) + } + return string(out), nil +} + +// GetHosts returns the output of the qhost command. +func (q *QHostImpl) GetHosts() ([]Host, error) { + out, err := q.NativeSpecification(nil) + if err != nil { + return nil, fmt.Errorf("failed to get output of qhost: %w", err) + } + hosts, err := ParseHosts(out) + if err != nil { + return nil, fmt.Errorf("failed to parse output of qhost: %w", err) + } + return hosts, nil +} + +// GetHostsFullMetrics returns the output of the qhost command with +// the -F option. +func (q *QHostImpl) GetHostsFullMetrics() ([]HostFullMetrics, error) { + out, err := q.NativeSpecification([]string{"-F"}) + if err != nil { + return nil, fmt.Errorf("failed to get output of qhost: %w", err) + } + hosts, err := ParseHostFullMetrics(out) + if err != nil { + return nil, fmt.Errorf("failed to parse output of qhost: %w", err) + } + return hosts, nil +} diff --git a/pkg/qhost/v9.0/qhost_suite_test.go b/pkg/qhost/v9.0/qhost_suite_test.go new file mode 100644 index 0000000..8594c59 --- /dev/null +++ b/pkg/qhost/v9.0/qhost_suite_test.go @@ -0,0 +1,13 @@ +package qhost_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestQhost(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Qhost Suite") +} diff --git a/pkg/qhost/v9.0/types.go b/pkg/qhost/v9.0/types.go new file mode 100644 index 0000000..cc86328 --- /dev/null +++ b/pkg/qhost/v9.0/types.go @@ -0,0 +1,87 @@ +/*___INFO__MARK_BEGIN__*/ +/************************************************************************* +* Copyright 2024 HPC-Gridware GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +************************************************************************/ +/*___INFO__MARK_END__*/ + +package qhost + +// Host is a struct that contains all values displayed by qhost output. +type Host struct { + Name string `json:"name"` + Arch string `json:"arch"` + NCPU int `json:"ncpu"` + NSOC int `json:"nsoc"` + NCOR int `json:"ncor"` + NTHR int `json:"nth"` + LOAD float64 `json:"load"` + MEMTOT int64 `json:"mem_total"` + MEMUSE int64 `json:"mem_used"` + SWAPTO int64 `json:"swap_total"` + SWAPUS int64 `json:"swap_used"` +} + +// HostFullMetrics is a struct that contains all values displayed by +// qhost -F output. +type HostFullMetrics struct { + Name string `json:"name"` + Arch string `json:"arch"` + NumProc float64 `json:"num_proc"` + MemTotal int64 `json:"mem_total"` + SwapTotal int64 `json:"swap_total"` + VirtualTotal int64 `json:"virtual_total"` + LoadAvg float64 `json:"load_avg"` + LoadShort float64 `json:"load_short"` + LoadMedium float64 `json:"load_medium"` + LoadLong float64 `json:"load_long"` + MemFree int64 `json:"mem_free"` + SwapFree int64 `json:"swap_free"` + VirtualFree int64 `json:"virtual_free"` + MemUsed int64 `json:"mem_used"` + SwapUsed int64 `json:"swap_used"` + VirtualUsed int64 `json:"virtual_used"` + CPU float64 `json:"cpu"` + Topology string `json:"topology"` + TopologyInuse string `json:"topology_inuse"` + Socket int64 `json:"socket"` + Core int64 `json:"core"` + Thread int64 `json:"thread"` + NPLoadAvg float64 `json:"np_load_avg"` + NPLoadShort float64 `json:"np_load_short"` + NPLoadMedium float64 `json:"np_load_medium"` + NPLoadLong float64 `json:"np_load_long"` + // Cluster defined metrics + Resources map[string]ResourceAvailability `json:"resources"` +} + +// ResourceAvailability is a struct that contains the availability of a resource +// on a host. +type ResourceAvailability struct { + Name string `json:"name"` + StringValue string `json:"value"` + FloatValue float64 `json:"float_value"` + // ResourceAvailabilityLimitedBy indices whether the resource availability + // is dominated by host "g" (global) or "l" (local). + ResourceAvailabilityLimitedBy string `json:"resource_availability_limited_by"` + // Source of the resource availability value: + // - "l" load value for a resource + // - "L" load value of a resource after an admin defined load scaling + // - "c" availabililty derived from the consumable calculation + // - "F" Non-consumable resource; Fixed value + Source string `json:"source"` + // The full output string "hl:np_load_medium=0.127500" + FullString string `json:"full_string"` +} diff --git a/pkg/qstat/v9.0/parser.go b/pkg/qstat/v9.0/parser.go index cdb30bc..690db20 100644 --- a/pkg/qstat/v9.0/parser.go +++ b/pkg/qstat/v9.0/parser.go @@ -25,8 +25,11 @@ import ( "log" "strconv" "strings" + "time" ) +const QstatDateFormat = "2006-01-02 03:04:05" + // ParseGroupByTask parses the input text into a slice of // SchedulerJobInfo instances (qstat -g t output). func ParseGroupByTask(input string) ([]ParallelJobTask, error) { @@ -92,6 +95,14 @@ func parseFixedWidthJobs(input string) ([]ParallelJobTask, error) { if strings.TrimSpace(line) == "" { continue } + // ignore lines with only dashes + if strings.HasPrefix(line, "---") { + continue + } + // ignore lines with description + if strings.HasPrefix(line, "job-ID") { + continue + } fields := make([]string, len(columnPositions)) for i, pos := range columnPositions { @@ -124,7 +135,9 @@ func parseFixedWidthJobs(input string) ([]ParallelJobTask, error) { task.Master = fields[7] } if len(fields) > 8 && fields[8] != "" { - task.JobInfo.TaskID = fields[8] + // could be a numer or something like "7-99:2" + task.JobInfo.JaTaskIDs = parseJaTaskIDs(fields[8]) + //task.JobInfo.TaskID = fields[8] } tasks = append(tasks, task) currentJob = &tasks[len(tasks)-1] @@ -138,6 +151,69 @@ func parseFixedWidthJobs(input string) ([]ParallelJobTask, error) { return tasks, nil } +// "7-99:2" or "1" to 7, 9, 11, 13, ... 99 or 1 +func parseJaTaskIDs(s string) []int64 { + if s == "" { + return []int64{} + } + + ids := []int64{} + + parts := strings.Split(s, ":") + // has a step + if len(parts) == 2 { + step, err := strconv.Atoi(parts[1]) + if err != nil { + return []int64{} + } + start := 0 + end := 0 + rangeParts := strings.Split(parts[0], "-") + if len(rangeParts) == 2 { + start, err = strconv.Atoi(rangeParts[0]) + if err != nil { + return []int64{} + } + end, err = strconv.Atoi(rangeParts[1]) + if err != nil { + return []int64{} + } + } else { + return []int64{} + } + for i := start; i <= end; i += step { + ids = append(ids, int64(i)) + } + return ids + } + + // no step, either number or range + + split := strings.Split(parts[0], "-") + // range + if len(split) == 2 { + start, err := strconv.Atoi(split[0]) + if err != nil { + return []int64{} + } + end, err := strconv.Atoi(split[1]) + if err != nil { + return []int64{} + } + for i := start; i <= end; i++ { + ids = append(ids, int64(i)) + } + } else { + id, err := strconv.Atoi(parts[0]) + if err != nil { + return []int64{} + } + ids = append(ids, int64(id)) + } + + return ids +} + func isContinuationLine(fields []string) bool { return len(fields[0]) == 0 } @@ -152,16 +228,25 @@ func parseFixedWidthJobInfo(fields []string) (*JobInfo, error) { if err != nil { return nil, fmt.Errorf("invalid priority: %v", err) } + submitTime, err := time.Parse(QstatDateFormat, fields[5]) + if err != nil { + return nil, fmt.Errorf("invalid submit time: %v", err) + } jobInfo := &JobInfo{ - JobID: jobID, - Priority: priority, - Name: fields[2], - User: fields[3], - State: fields[4], - SubmitStartAt: fields[5], - Queue: fields[6], - Slots: 1, + JobID: jobID, + Priority: priority, + Name: fields[2], + User: fields[3], + State: fields[4], + Queue: fields[6], + Slots: 1, + } + if strings.Contains(jobInfo.State, "r") { + jobInfo.StartTime = submitTime + } + if strings.Contains(jobInfo.State, "q") { + jobInfo.SubmitTime = submitTime } return jobInfo, nil @@ -256,3 +341,428 @@ func parseJob(block string) (SchedulerJobInfo, error) { } return info, nil } + +/* + qstat -ext +job-ID prior ntckts name user project department state cpu mem io tckts ovrts otckt ftckt stckt share queue slots ja-task-ID +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + 31 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 + 32 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 1 + 32 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 3 + 32 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 5 + 32 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 7 + 32 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 9 +*/ + +func ParseExtendedJobInfo(output string) ([]ExtendedJobInfo, error) { + ext := []ExtendedJobInfo{} + + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if strings.HasPrefix(line, "job-ID") { + continue + } + if strings.HasPrefix(line, "-------------------") { + continue + } + info, err := parseExtendedJobInfoLine(line) + if err != nil { + return nil, err + } + ext = append(ext, info) + } + + return ext, nil +} + +/* +qstat -ext +job-ID prior ntckts name user project department state cpu mem io tckts ovrts otckt ftckt stckt share queue slots ja-task-ID +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 1 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 2 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 3 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 4 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 5 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 6 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 7 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 8 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 9 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 10 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 11 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 12 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 13 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 14 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 15 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 16 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 17 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 18 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 19 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 20 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 21 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 22 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 23 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 24 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 25 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 26 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 27 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 28 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 29 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 30 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 31 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 32 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 33 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 34 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 35 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 36 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 37 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 38 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 39 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 40 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 41 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 42 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 43 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 44 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 45 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 46 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 47 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 48 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 49 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 50 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 51 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 52 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 53 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 54 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 55 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 56 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 57 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 58 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 59 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 60 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 61 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 62 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 63 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 64 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 65 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 66 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 67 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 68 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 69 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 70 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 71 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 72 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 73 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 74 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 75 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 76 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 77 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 78 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 79 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 80 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 81 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 82 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 83 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 84 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 85 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 86 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 87 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 88 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 89 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 90 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 91 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 92 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 93 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 94 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 95 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 96 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 97 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 98 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 99 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 100 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 1 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 2 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 3 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 4 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 5 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 6 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 7 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 8 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 9 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 10 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 11 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 12 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 13 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 14 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 15 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 16 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 17 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 18 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 19 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 20 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 21 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 22 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 23 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 24 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 25 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 26 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 27 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 28 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 29 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 30 + 34 0.55500 0.50000 sleep root NA defaultdep qw 0 0 0 0 0 0.00 1 31-100:1 + 35 0.55500 0.50000 sleep root NA defaultdep qw 0 0 0 0 0 0.00 1 +*/ + +func parseExtendedJobInfoLine(line string) (ExtendedJobInfo, error) { + fields := strings.Fields(line) + + // Initialize variables + var jobID int + var prior float64 + var ntckts float64 + var name, user, project, department, state, cpu, mem, io string + var tckts, ovrts, otckt, ftckt, stckt int + var share float64 + var queue string + var slots int + var jaTaskID string + + fmt.Println(len(fields)) + + if len(fields) == 19 || len(fields) == 20 { + // Expected number of fields when job is in 'r' state + var err error + jobID, err = strconv.Atoi(fields[0]) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse jobID: %v", err) + } + prior, err = strconv.ParseFloat(fields[1], 64) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse prior: %v", err) + } + ntckts, err = strconv.ParseFloat(fields[2], 64) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse ntckts: %v", err) + } + name = fields[3] + user = fields[4] + project = fields[5] + department = fields[6] + state = fields[7] + cpu = fields[8] + mem = fields[9] + io = fields[10] + tckts, err = strconv.Atoi(fields[11]) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse tckts: %v", err) + } + ovrts, err = strconv.Atoi(fields[12]) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse ovrts: %v", err) + } + otckt, err = strconv.Atoi(fields[13]) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse otckt: %v", err) + } + ftckt, err = strconv.Atoi(fields[14]) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse ftckt: %v", err) + } + stckt, err = strconv.Atoi(fields[15]) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse stckt: %v", err) + } + share, err = strconv.ParseFloat(fields[16], 64) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse share: %v", err) + } + queue = fields[17] + slots, err = strconv.Atoi(fields[18]) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse slots: %v", err) + } + if len(fields) == 20 { + jaTaskID = fields[19] + } + } else if len(fields) == 16 || len(fields) == 15 { + // Expected number of fields when job is in 'qw' state + var err error + jobID, err = strconv.Atoi(fields[0]) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse jobID: %v", err) + } + prior, err = strconv.ParseFloat(fields[1], 64) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse prior: %v", err) + } + ntckts, err = strconv.ParseFloat(fields[2], 64) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse ntckts: %v", err) + } + name = fields[3] + user = fields[4] + project = fields[5] + department = fields[6] + state = fields[7] + // cpu, mem, io are missing; set to default values + cpu = "" + mem = "" + io = "" + tckts, err = strconv.Atoi(fields[8]) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse tckts: %v", err) + } + ovrts, err = strconv.Atoi(fields[9]) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse ovrts: %v", err) + } + otckt, err = strconv.Atoi(fields[10]) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse otckt: %v", err) + } + ftckt, err = strconv.Atoi(fields[11]) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse ftckt: %v", err) + } + stckt, err = strconv.Atoi(fields[12]) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse stckt: %v", err) + } + share, err = strconv.ParseFloat(fields[13], 64) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse share: %v", err) + } + // 'queue' is missing in 'qw' state; set to default value + queue = "" + slots, err = strconv.Atoi(fields[14]) + if err != nil { + return ExtendedJobInfo{}, fmt.Errorf("failed to parse slots: %v", err) + } + if len(fields) == 16 { + jaTaskID = fields[15] + } + } else { + return ExtendedJobInfo{}, fmt.Errorf("unexpected number of fields: %d (%s)", + len(fields), line) + } + + // TODO convert them correctly + mem = io + io = mem + + return ExtendedJobInfo{ + JobID: jobID, + Priority: prior, + Name: name, + User: user, + Project: project, + Department: department, + State: state, + CPU: cpu, + //Memory: mem, + //IO: io, + Tckts: tckts, + Ovrts: ovrts, + Otckt: otckt, + Ftckt: ftckt, + Stckt: stckt, + Ntckts: ntckts, + Share: share, + Queue: queue, + Slots: slots, + JATaskID: jaTaskID, + }, nil +} + +/* +job-ID prior name user state submit/start at queue slots ja-task-ID +----------------------------------------------------------------------------------------------------------------- + + 8 0.55500 sleep root r 2024-12-14 17:21:43 all.q@master 1 + 9 0.55500 sleep root r 2024-12-14 17:21:44 all.q@master 1 + 10 0.55500 sleep root r 2024-12-14 17:21:44 all.q@master 1 + 11 0.55500 sleep root r 2024-12-14 17:21:45 all.q@master 1 + 12 0.55500 sleep root qw 2024-12-14 17:21:45 1 + 13 0.55500 sleep root qw 2024-12-14 17:21:46 1 + 14 0.55500 sleep root qw 2024-12-14 17:21:47 1 + 15 0.55500 sleep root qw 2024-12-14 17:22:00 1 1-99:2 +*/ +func ParseJobInfo(out string) ([]JobInfo, error) { + lines := strings.Split(out, "\n") + jobInfos := make([]JobInfo, 0, len(lines)-3) + for _, line := range lines { + if strings.HasPrefix(line, "job-ID") { + continue + } + if strings.HasPrefix(line, "---------") { + continue + } + fields := strings.Fields(line) + if len(fields) < 8 { + continue + } + jobID, err := strconv.Atoi(fields[0]) + if err != nil { + return nil, fmt.Errorf("failed to parse jobID: %v", err) + } + priority, err := strconv.ParseFloat(fields[1], 64) + if err != nil { + return nil, fmt.Errorf("failed to parse priority: %v", err) + } + name := fields[2] + user := fields[3] + state := fields[4] + + var submitTime time.Time + var startTime time.Time + + var queue string + var slots int + var jaTaskIDs []int64 + // the state defines the format of the rest of the fields + if state == "r" { + // we have a running job in a queue intance + submitTimeString := fields[5] + fields[6] + submitTime, err = time.Parse("2024-12-14 17:21:43", submitTimeString) + if err != nil { + return nil, fmt.Errorf("failed to parse submit time: %v", err) + } + queue = fields[7] + slots, err = strconv.Atoi(fields[8]) + if err != nil { + return nil, fmt.Errorf("failed to parse slots: %v", err) + } + // TODO parse jaTaskIDs + } else if state == "qw" { + // we have a queued job + startTimeString := fields[5] + fields[6] + startTime, err = time.Parse("2024-12-14 17:21:43", startTimeString) + if err != nil { + return nil, fmt.Errorf("failed to parse run time: %v", err) + } + slots, err = strconv.Atoi(fields[7]) + if err != nil { + return nil, fmt.Errorf("failed to parse slots: %v", err) + } + // TODO parse jaTaskIDs + } + + jobInfo := JobInfo{ + JobID: jobID, + Priority: priority, + Name: name, + User: user, + State: state, + SubmitTime: submitTime, + StartTime: startTime, + Queue: queue, + Slots: slots, + JaTaskIDs: jaTaskIDs, + } + jobInfos = append(jobInfos, jobInfo) + } + return jobInfos, nil +} diff --git a/pkg/qstat/v9.0/parser_test.go b/pkg/qstat/v9.0/parser_test.go index 093ce7a..7ffb408 100644 --- a/pkg/qstat/v9.0/parser_test.go +++ b/pkg/qstat/v9.0/parser_test.go @@ -1,6 +1,27 @@ +/*___INFO__MARK_BEGIN__*/ +/************************************************************************* +* Copyright 2024 HPC-Gridware GmbH +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +************************************************************************/ +/*___INFO__MARK_END__*/ + package qstat_test import ( + "time" + qstat "github.com/hpc-gridware/go-clusterscheduler/pkg/qstat/v9.0" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -27,40 +48,74 @@ var _ = Describe("Parser", func() { Expect(jobs[0].Name).To(Equal("sleep")) Expect(jobs[0].User).To(Equal("root")) Expect(jobs[0].State).To(Equal("r")) - Expect(jobs[0].SubmitStartAt).To(Equal("2024-10-28 07:21:41")) + // "2024-10-28 07:21:41" + Expect(jobs[0].StartTime.Year()).To(Equal(2024)) + Expect(jobs[0].StartTime.Month()).To(Equal(time.October)) + Expect(jobs[0].StartTime.Day()).To(Equal(28)) + Expect(jobs[0].StartTime.Hour()).To(Equal(7)) + Expect(jobs[0].StartTime.Minute()).To(Equal(21)) + Expect(jobs[0].StartTime.Second()).To(Equal(41)) Expect(jobs[0].Queue).To(Equal("all.q@master")) Expect(jobs[0].Master).To(Equal("MASTER")) - Expect(jobs[0].TaskID).To(Equal("")) + Expect(len(jobs[0].JaTaskIDs)).To(Equal(0)) Expect(jobs[1].JobID).To(Equal(15)) Expect(jobs[1].Name).To(Equal("sleep")) Expect(jobs[1].User).To(Equal("root")) Expect(jobs[1].State).To(Equal("r")) - Expect(jobs[1].SubmitStartAt).To(Equal("2024-10-28 07:26:14")) - Expect(jobs[1].TaskID).To(Equal("1")) + // "2024-10-28 07:26:14" + Expect(jobs[1].StartTime.Year()).To(Equal(2024)) + Expect(jobs[1].StartTime.Month()).To(Equal(time.October)) + Expect(jobs[1].StartTime.Day()).To(Equal(28)) + Expect(jobs[1].StartTime.Hour()).To(Equal(7)) + Expect(jobs[1].StartTime.Minute()).To(Equal(26)) + Expect(jobs[1].StartTime.Second()).To(Equal(14)) + Expect(jobs[1].JaTaskIDs).To(Equal([]int64{1})) Expect(jobs[2].JobID).To(Equal(15)) Expect(jobs[2].Name).To(Equal("sleep")) Expect(jobs[2].User).To(Equal("root")) Expect(jobs[2].State).To(Equal("r")) - Expect(jobs[2].SubmitStartAt).To(Equal("2024-10-28 07:26:14")) - Expect(jobs[2].TaskID).To(Equal("3")) + // "2024-10-28 07:26:14" + Expect(jobs[2].StartTime.Year()).To(Equal(2024)) + Expect(jobs[2].StartTime.Month()).To(Equal(time.October)) + Expect(jobs[2].StartTime.Day()).To(Equal(28)) + Expect(jobs[2].StartTime.Hour()).To(Equal(7)) + Expect(jobs[2].StartTime.Minute()).To(Equal(26)) + Expect(jobs[2].StartTime.Second()).To(Equal(14)) + Expect(jobs[2].JaTaskIDs).To(Equal([]int64{3})) Expect(jobs[3].JobID).To(Equal(15)) Expect(jobs[3].Name).To(Equal("sleep")) Expect(jobs[3].User).To(Equal("root")) Expect(jobs[3].State).To(Equal("r")) - Expect(jobs[3].SubmitStartAt).To(Equal("2024-10-28 07:26:14")) - Expect(jobs[3].TaskID).To(Equal("5")) + // "2024-10-28 07:26:14" + Expect(jobs[3].StartTime.Year()).To(Equal(2024)) + Expect(jobs[3].StartTime.Month()).To(Equal(time.October)) + Expect(jobs[3].StartTime.Day()).To(Equal(28)) + Expect(jobs[3].StartTime.Hour()).To(Equal(7)) + Expect(jobs[3].StartTime.Minute()).To(Equal(26)) + Expect(jobs[3].StartTime.Second()).To(Equal(14)) + Expect(jobs[3].JaTaskIDs).To(Equal([]int64{5})) Expect(jobs[4].JobID).To(Equal(17)) Expect(jobs[4].Name).To(Equal("sleep")) Expect(jobs[4].User).To(Equal("root")) Expect(jobs[4].State).To(Equal("qw")) - Expect(jobs[4].SubmitStartAt).To(Equal("2024-10-28 07:27:50")) - Expect(jobs[4].TaskID).To(Equal("")) + // "2024-10-28 07:27:50" + Expect(jobs[4].SubmitTime.Year()).To(Equal(2024)) + Expect(jobs[4].SubmitTime.Month()).To(Equal(time.October)) + Expect(jobs[4].SubmitTime.Day()).To(Equal(28)) + Expect(jobs[4].SubmitTime.Hour()).To(Equal(7)) + Expect(jobs[4].SubmitTime.Minute()).To(Equal(27)) + Expect(jobs[4].SubmitTime.Second()).To(Equal(50)) Expect(jobs[5].JobID).To(Equal(12)) Expect(jobs[5].Name).To(Equal("sleep")) Expect(jobs[5].User).To(Equal("root")) Expect(jobs[5].State).To(Equal("qw")) - Expect(jobs[5].SubmitStartAt).To(Equal("2024-10-28 07:17:34")) - Expect(jobs[5].TaskID).To(Equal("")) + // "2024-10-28 07:17:34" + Expect(jobs[5].SubmitTime.Year()).To(Equal(2024)) + Expect(jobs[5].SubmitTime.Month()).To(Equal(time.October)) + Expect(jobs[5].SubmitTime.Day()).To(Equal(28)) + Expect(jobs[5].SubmitTime.Hour()).To(Equal(7)) + Expect(jobs[5].SubmitTime.Minute()).To(Equal(17)) + Expect(jobs[5].SubmitTime.Second()).To(Equal(34)) }) It("should parse the output of qstat -g t", func() { @@ -169,14 +224,198 @@ var _ = Describe("Parser", func() { Expect(jobs[40].Name).To(Equal("sleep")) Expect(jobs[40].User).To(Equal("root")) Expect(jobs[40].State).To(Equal("qw")) - Expect(jobs[40].SubmitStartAt).To(Equal("2024-10-28 07:17:34")) + Expect(jobs[40].SubmitTime.Format(qstat.QstatDateFormat)).To(Equal("2024-10-28 07:17:34")) // job before last Expect(jobs[39].JobID).To(Equal(20)) - Expect(jobs[39].TaskID).To(Equal("2")) Expect(jobs[39].Queue).To(Equal("all.q@sim9")) Expect(jobs[39].Master).To(Equal("SLAVE")) - Expect(jobs[39].SubmitStartAt).To(Equal("2024-10-28 08:34:36")) + Expect(jobs[39].StartTime.Format(qstat.QstatDateFormat)).To(Equal("2024-10-28 08:34:36")) + }) + + }) + + /* + + job-ID prior name user state submit/start at queue master ja-task-ID + ------------------------------------------------------------------------------------------------------------------ + 14 0.50500 sleep root r 2024-10-28 07:21:41 all.q@master MASTER + 15 0.50500 sleep root r 2024-10-28 07:26:14 all.q@master MASTER 1 + 15 0.50500 sleep root r 2024-10-28 07:26:14 all.q@master MASTER 3 + 15 0.50500 sleep root r 2024-10-28 07:26:14 all.q@master MASTER 5 + 17 0.60500 sleep root qw 2024-10-28 07:27:50 + 12 0.50500 sleep root qw 2024-10-28 07:17:34 + 15 0.50500 sleep root qw 2024-10-28 07:26:14 7-99:2 + */ + + Context("ParseQstatExt", func() { + + It("should parse the output of qstat -ext", func() { + + input := `job-ID prior ntckts name user project department state cpu mem io tckts ovrts otckt ftckt stckt share queue slots ja-task-ID +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + 36 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 + 37 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 +` + + jobs, err := qstat.ParseExtendedJobInfo(input) + Expect(err).NotTo(HaveOccurred()) + Expect(len(jobs)).To(Equal(2)) + Expect(jobs[0].JobID).To(Equal(36)) + Expect(jobs[0].Name).To(Equal("sleep")) + Expect(jobs[0].User).To(Equal("root")) + Expect(jobs[0].State).To(Equal("r")) + Expect(jobs[0].Queue).To(Equal("all.q@sim10")) + Expect(jobs[0].Slots).To(Equal(1)) + + input2 := `job-ID prior ntckts name user project department state cpu mem io tckts ovrts otckt ftckt stckt share queue slots ja-task-ID +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 1 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 2 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 3 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 4 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 5 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 6 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 7 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 8 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 9 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 10 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 11 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 12 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 13 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 14 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 15 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 16 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 17 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 18 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 19 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 20 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 21 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 22 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 23 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 24 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 25 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 26 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 27 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 28 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 29 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 30 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 31 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 32 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 33 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 34 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 35 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 36 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 37 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 38 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 39 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 40 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 41 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 42 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 43 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 44 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 45 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 46 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 47 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 48 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 49 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 50 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 51 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 52 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 53 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 54 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 55 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 56 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 57 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 58 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 59 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 60 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 61 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 62 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 63 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 64 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 65 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 66 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 67 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 68 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 69 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 70 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 71 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 72 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 73 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 74 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 75 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 76 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 77 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 78 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 79 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 80 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 81 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 82 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 83 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 84 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 85 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 86 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 87 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 88 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 89 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 90 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 91 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 92 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 93 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 94 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 95 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 96 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 97 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 98 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 99 + 33 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 100 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 1 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 2 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 3 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 4 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 5 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 6 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 7 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 8 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 9 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 10 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 11 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 12 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 13 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 14 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 15 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 16 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 17 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim1 1 18 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim6 1 19 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim8 1 20 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim12 1 21 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim3 1 22 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim4 1 23 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim5 1 24 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim2 1 25 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@master 1 26 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim11 1 27 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim10 1 28 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim9 1 29 + 34 0.55500 0.50000 sleep root NA defaultdep r NA NA NA 0 0 0 0 0 0.00 all.q@sim7 1 30 + 34 0.55500 0.50000 sleep root NA defaultdep qw 0 0 0 0 0 0.00 1 31-100:1 + 35 0.55500 0.50000 sleep root NA defaultdep qw 0 0 0 0 0 0.00 1 +` + + jobs, err = qstat.ParseExtendedJobInfo(input2) + Expect(err).NotTo(HaveOccurred()) + Expect(len(jobs)).To(Equal(132)) + Expect(jobs[130].JATaskID).To(Equal("31-100:1")) + Expect(jobs[131].JobID).To(Equal(35)) + Expect(jobs[131].Name).To(Equal("sleep")) + Expect(jobs[131].User).To(Equal("root")) + Expect(jobs[131].State).To(Equal("qw")) + Expect(jobs[131].Priority).To(Equal(0.555)) + Expect(jobs[131].Ntckts).To(Equal(0.5)) + Expect(jobs[131].User).To(Equal("root")) + Expect(jobs[131].Slots).To(Equal(1)) }) }) diff --git a/pkg/qstat/v9.0/qstat.go b/pkg/qstat/v9.0/qstat.go index 045c1b7..9fd2ff0 100644 --- a/pkg/qstat/v9.0/qstat.go +++ b/pkg/qstat/v9.0/qstat.go @@ -36,7 +36,7 @@ type QStat interface { // and returns the raw output NativeSpecification(args []string) (string, error) // qstat -ext - ShowAdditionalAttributes() ([]ExtendedJobInfo, error) + ShowJobsWithAdditionalAttributes() ([]ExtendedJobInfo, error) // qstat -explain a|c|A|E ShowQueueExplanation(reason string) ([]QueueExplanation, error) // qstat -f diff --git a/pkg/qstat/v9.0/qstat_impl.go b/pkg/qstat/v9.0/qstat_impl.go index 3a98eb5..9e7ee5f 100644 --- a/pkg/qstat/v9.0/qstat_impl.go +++ b/pkg/qstat/v9.0/qstat_impl.go @@ -141,8 +141,20 @@ func (q *QStatImpl) NativeSpecification(args []string) (string, error) { return string(out), nil } -func (q *QStatImpl) ShowAdditionalAttributes() ([]ExtendedJobInfo, error) { - return nil, fmt.Errorf("not implemented") +func (q *QStatImpl) ShowJobs() ([]JobInfo, error) { + out, err := q.NativeSpecification(nil) + if err != nil { + return nil, fmt.Errorf("failed to get output of qstat: %w", err) + } + return ParseJobInfo(out) +} + +func (q *QStatImpl) ShowJobsWithAdditionalAttributes() ([]ExtendedJobInfo, error) { + out, err := q.NativeSpecification([]string{"-ext"}) + if err != nil { + return nil, fmt.Errorf("failed to get output of qstat: %w", err) + } + return ParseExtendedJobInfo(out) } func (q *QStatImpl) ShowQueueExplanation(reason string) ([]QueueExplanation, error) { diff --git a/pkg/qstat/v9.0/types.go b/pkg/qstat/v9.0/types.go index dd4c511..2e86ddf 100644 --- a/pkg/qstat/v9.0/types.go +++ b/pkg/qstat/v9.0/types.go @@ -19,16 +19,19 @@ package qstat +import "time" + type JobInfo struct { - JobID int `json:"job_id"` - Priority float64 `json:"prior"` - Name string `json:"name"` - User string `json:"user"` - State string `json:"state"` - SubmitStartAt string `json:"submit_start_at"` - Queue string `json:"queue"` - Slots int `json:"slots"` - TaskID string `json:"ja_task_id"` + JobID int `json:"job_id"` + Priority float64 `json:"prior"` + Name string `json:"name"` + User string `json:"user"` + State string `json:"state"` + SubmitTime time.Time `json:"submit_start_at"` + StartTime time.Time `json:"start_time"` + Queue string `json:"queue"` + Slots int `json:"slots"` + JaTaskIDs []int64 `json:"ja_task_ids"` } type ExtendedJobInfo struct {