diff --git a/go.mod b/go.mod index af23cd56..253edf3d 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.9 github.com/spf13/viper v1.10.1 - github.com/stackql-labs/omnisdk v0.1.0-alpha07 + github.com/stackql-labs/omnisdk v0.1.1-alpha06 github.com/stackql/any-sdk v0.5.4-alpha01 github.com/stackql/go-suffix-map v0.0.1-alpha01 github.com/stackql/psql-wire v0.1.2-beta01 diff --git a/go.sum b/go.sum index 02190eb0..dfe50192 100644 --- a/go.sum +++ b/go.sum @@ -348,8 +348,8 @@ github.com/spf13/viper v1.10.1 h1:nuJZuYpG7gTj/XqiUwg8bA0cp1+M2mC3J4g5luUYBKk= github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8qy1rU= github.com/spiffe/go-spiffe/v2 v2.7.0 h1:uXe1MflJoHw58wAUvxVlcM7WpKtijWG7I1UidcGh6g4= github.com/spiffe/go-spiffe/v2 v2.7.0/go.mod h1:47Q0Q9/AqGha8QLHp+kxpH4Wca7X7EnOtlIJy3mxZ3U= -github.com/stackql-labs/omnisdk v0.1.0-alpha07 h1:kGUtupkwAVZWwCkMQZ4vjCD5jIDPmGCSq/OdH4Dmos0= -github.com/stackql-labs/omnisdk v0.1.0-alpha07/go.mod h1:SHUOryRWXeZip4JTWeA8iwmz4lYdBSp0nbWXDX29JrU= +github.com/stackql-labs/omnisdk v0.1.1-alpha06 h1:jVetiuZa6J1uWJ4vPc/YQtol4l7Lgoq281UWvYXw6GM= +github.com/stackql-labs/omnisdk v0.1.1-alpha06/go.mod h1:WzvNj/bVv53yGFsVJpYWCJC1xAEdmQSFJl9eVpkRpCY= github.com/stackql/any-sdk v0.5.4-alpha01 h1:AyjD2Hyk7D8v1fHHtLpfeqQRxKuj7rC71dzi1Zh9LoA= github.com/stackql/any-sdk v0.5.4-alpha01/go.mod h1:BiE8uiAJMUa8n4U/yMlhvrVhE2M+5Rt8utBXYkwj9To= github.com/stackql/go-suffix-map v0.0.1-alpha01 h1:TDUDS8bySu41Oo9p0eniUeCm43mnRM6zFEd6j6VUaz8= diff --git a/internal/stackql/handler/handler.go b/internal/stackql/handler/handler.go index d9157527..06625bef 100644 --- a/internal/stackql/handler/handler.go +++ b/internal/stackql/handler/handler.go @@ -323,6 +323,12 @@ func (hc *standardHandlerContext) GetSupportedProviders(extended bool) (map[stri } else { retVal[pn] = getProviderMap(pn, pd) } + // Every locally available bundle is also addressable document-first. + unstable := intrinsic.UnstablePrefix + pn + retVal[unstable] = map[string]interface{}{ + "name": unstable, + "version": intrinsic.ProviderVersion, + } } return retVal, nil } diff --git a/internal/stackql/intrinsic/doc.go b/internal/stackql/intrinsic/doc.go new file mode 100644 index 00000000..ce91cc56 --- /dev/null +++ b/internal/stackql/intrinsic/doc.go @@ -0,0 +1,237 @@ +package intrinsic + +import ( + "context" + "fmt" + "path/filepath" + "sort" + "strings" + + "github.com/stackql-labs/omnisdk/pkg/docparse/aot" + "github.com/stackql-labs/omnisdk/pkg/omnisdk" + "github.com/stackql/any-sdk/public/formulation" + "github.com/stackql/psql-wire/pkg/sqldata" + "github.com/stackql/stackql/internal/stackql/internal_data_transfer/internaldto" + "github.com/stackql/stackql/internal/stackql/util" + + "github.com/stackql/stackql-parser/go/vt/sqlparser" +) + +// UnstablePrefix names the document-driven providers. The convention is +// omnisdk's own, and its addresses carry the prefixed provider name, so this is +// its constant rather than a copy of the literal. +const UnstablePrefix = aot.DefaultProviderPrefix + +// docProvider is the bundle behind an unstable provider name, or false. +func docProvider(name string) (string, bool) { + trimmed := strings.TrimSpace(name) + if !strings.HasPrefix(strings.ToLower(trimmed), UnstablePrefix) { + return "", false + } + bundle := trimmed[len(UnstablePrefix):] + if bundle == "" { + return "", false + } + return bundle, true +} + +// bundleAliases maps the provider name stackql presents onto the directory the +// registry actually wrote. Keeping them apart matters: the name a caller typed +// is the one echoed back, and it is the only one they can address. +var bundleAliases = map[string]string{ //nolint:gochecknoglobals // fixed mapping + "google": "googleapis.com", +} + +func bundleDir(label string) string { + if dir, ok := bundleAliases[strings.ToLower(label)]; ok { + return dir + } + return label +} + +// docRoot is the bundle's own directory inside stackql's registry root. The +// versioned directory is used rather than the root itself, because a registry +// root is addressed as ".." and a provider whose +// name carries a dot ("googleapis.com") cannot be named that way. +func docRoot(ctx queryContext, bundle string) (string, error) { + root := filepath.Join(ctx.GetRuntimeContext().ApplicationFilesRootPath, "src", bundleDir(bundle)) + matches, err := filepath.Glob(filepath.Join(root, "*", "provider.yaml")) + if err != nil || len(matches) == 0 { + return "", fmt.Errorf("no provider document for '%s%s' under '%s'", UnstablePrefix, bundle, root) + } + sort.Strings(matches) + return filepath.Dir(matches[len(matches)-1]), nil +} + +// docServices lists the services a bundle ships documents for. +func docServices(ctx queryContext, bundle string) ([]string, error) { + dir, err := docRoot(ctx, bundle) + if err != nil { + return nil, err + } + services, _, err := omnisdk.DocCatalog(dir, bundle) + return services, err +} + +// docResourceTables presents a service's resources as relations. +func docResourceTables(ctx queryContext, bundle, service string) ([]table, error) { + dir, err := docRoot(ctx, bundle) + if err != nil { + return nil, err + } + resources, err := omnisdk.DocResources(dir, bundle, service) + if err != nil { + return nil, err + } + sort.Strings(resources) + out := make([]table, 0, len(resources)) + for _, resource := range resources { + out = append(out, table{service: service, name: resource, isData: true}) + } + return out, nil +} + +// docMethods lists a resource's methods as the document declares them. +func docMethods(ctx queryContext, bundle, service, resource string) ([]relationMethod, error) { + dir, err := docRoot(ctx, bundle) + if err != nil { + return nil, err + } + methods, err := omnisdk.DocMethods(dir, bundle, service, resource) + if err != nil { + return nil, err + } + out := make([]relationMethod, 0, len(methods)) + for _, method := range methods { + out = append(out, relationMethod{name: method.Name, description: method.OperationID}) + } + return out, nil +} + +// docSelectFunc routes a SELECT over a document-driven relation. The address is +// the bundle's own "..", and the plan it yields +// streams exactly as a hand-authored one does. +func docSelectFunc( + ctx queryContext, + node *sqlparser.Select, + bundle, service, resource string, +) (func() internaldto.ExecutorOutput, bool) { + if unsupported := unsupportedClauses(node); len(unsupported) > 0 { + return refuse(fmt.Errorf( + "relation '%s%s.%s.%s' streams its rows, so %s cannot be applied; remove %s from the query", + UnstablePrefix, bundle, service, resource, + strings.Join(unsupported, ", "), pluralClause(len(unsupported)))), true + } + params, badPredicates := equalityPredicates(node.Where) + if len(badPredicates) > 0 { + return refuse(fmt.Errorf( + "relation '%s%s.%s.%s' streams its rows, so only equality predicates are applied; "+ + "%s cannot be honoured", + UnstablePrefix, bundle, service, resource, strings.Join(badPredicates, ", "))), true + } + address := fmt.Sprintf("%s%s.%s.%s", UnstablePrefix, bundle, service, resource) + return func() internaldto.ExecutorOutput { + input := previewCfg + dir, dirErr := docRoot(ctx, bundle) + if dirErr != nil { + return internaldto.NewErroneousExecutorOutput(dirErr) + } + plan, err := omnisdk.NewFromCatalog(dir, address, omnisdk.Args{ + Params: params, + Auth: omnisdkAuth(providerAuthContext(ctx, bundle)), + Endpoint: input.getEndpoint(), + InsecureSkipTLSVerify: input.getInsecureSkipTLSVerify(), + }) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + rows, openErr := plan.Open(context.Background()) + if openErr != nil { + return internaldto.NewErroneousExecutorOutput(openErr) + } + // A document declares no egress schema, so the columns are those the + // first row carries; projection is applied over them. + stream := &rowStream{ + rows: rows, + batchSize: input.getBatchSize(), + flushInterval: input.getFlushInterval(), + table: sqldata.NewSQLTable(0, resource), + typCfg: ctx.GetTypingConfig(), + projection: node.SelectExprs, + } + primed, readErr := newPrimedStream(stream) + if readErr != nil { + return internaldto.NewErroneousExecutorOutput(readErr) + } + return internaldto.NewExecutorOutput(primed, nil, nil, nil, nil) + }, true +} + +func refuse(err error) func() internaldto.ExecutorOutput { + return func() internaldto.ExecutorOutput { + return internaldto.NewErroneousExecutorOutput(err) + } +} + +func showDocServices(ctx queryContext, bundle string, extended bool) internaldto.ExecutorOutput { + services, err := docServices(ctx, bundle) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + rows := make(map[string]map[string]interface{}, len(services)) + for i, service := range services { + row := map[string]interface{}{"id": service, "name": service, "title": service} + if extended { + row["description"] = service + row["version"] = ProviderVersion + row["preferred"] = nil + } + rows[fmt.Sprintf("%06d", i)] = row + } + return prepare(ctx, formulation.GetServicesHeader(extended), rows, util.DefaultRowSort) +} + +func showDocResources( + ctx queryContext, bundle, service string, extended bool) internaldto.ExecutorOutput { + tables, err := docResourceTables(ctx, bundle, service) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + rows := make(map[string]map[string]interface{}, len(tables)) + for i, tbl := range tables { + row := map[string]interface{}{ + "id": fmt.Sprintf("%s%s.%s.%s", UnstablePrefix, bundle, service, tbl.name), + "name": tbl.name, + } + if extended { + row["description"] = tbl.description + } + rows[fmt.Sprintf("%06d", i)] = row + } + return prepare(ctx, formulation.GetResourcesHeader(extended), rows, util.DefaultRowSort) +} + +func showDocMethods( + ctx queryContext, bundle, service, resource string, extended bool) internaldto.ExecutorOutput { + methods, err := docMethods(ctx, bundle, service, resource) + if err != nil { + return internaldto.NewErroneousExecutorOutput(err) + } + columnOrder := []string{"MethodName", "RequiredParams", "SQLVerb"} + if extended { + columnOrder = append(columnOrder, "description") + } + rows := make(map[string]map[string]interface{}, len(methods)) + for i, method := range methods { + row := map[string]interface{}{ + "MethodName": method.name, + "RequiredParams": strings.Join(method.requiredParams, ", "), + "SQLVerb": strings.ToUpper(selectMethodName), + } + if extended { + row["description"] = method.description + } + rows[fmt.Sprintf("%06d", i)] = row + } + return prepare(ctx, columnOrder, rows, util.DefaultRowSort) +} diff --git a/internal/stackql/intrinsic/intrinsic.go b/internal/stackql/intrinsic/intrinsic.go index 213408b6..72a2daf7 100644 --- a/internal/stackql/intrinsic/intrinsic.go +++ b/internal/stackql/intrinsic/intrinsic.go @@ -35,6 +35,7 @@ type column struct { type table struct { service string name string + isData bool description string columns []column } @@ -48,6 +49,7 @@ type queryContext interface { SetCurrentProvider(string) GetTypingConfig() typing.Config GetAuthContext(providerName string) (*dto.AuthCtx, error) + GetRuntimeContext() dto.RuntimeCtx } func GeneratePrimitiveFunc( @@ -76,7 +78,11 @@ func GenerateStreamFunc( } func IsProvider(name string) bool { - return strings.EqualFold(strings.TrimSpace(name), ProviderName) + if strings.EqualFold(strings.TrimSpace(name), ProviderName) { + return true + } + _, isDoc := docProvider(name) + return isDoc } func resolveProvider(providerName string, currentProvider string) string { @@ -132,17 +138,34 @@ func showFunc( extended := isExtended(node.Extended) switch strings.ToUpper(strings.TrimSpace(node.Type)) { case "SERVICES": - if !IsProvider(resolveProvider(node.OnTable.Name.GetRawVal(), currentProvider)) { + provider := resolveProvider(node.OnTable.Name.GetRawVal(), currentProvider) + if bundle, isDoc := docProvider(provider); isDoc { + return func() internaldto.ExecutorOutput { return showDocServices(ctx, bundle, extended) }, true + } + if !IsProvider(provider) { return nil, false } return func() internaldto.ExecutorOutput { return showServices(ctx, extended) }, true case "RESOURCES": serviceStr := node.OnTable.Name.GetRawVal() + if bundle, isDoc := docProvider(resolveProvider(node.OnTable.Qualifier.GetRawVal(), currentProvider)); isDoc { + return func() internaldto.ExecutorOutput { + return showDocResources(ctx, bundle, serviceStr, extended) + }, true + } if !isService(node.OnTable.Qualifier.GetRawVal(), serviceStr, currentProvider) { return nil, false } return func() internaldto.ExecutorOutput { return showResources(ctx, serviceStr, extended) }, true case "METHODS": + if bundle, isDoc := docProvider( + resolveProvider(node.OnTable.QualifierSecond.GetRawVal(), currentProvider)); isDoc { + service := node.OnTable.Qualifier.GetRawVal() + resource := node.OnTable.Name.GetRawVal() + return func() internaldto.ExecutorOutput { + return showDocMethods(ctx, bundle, service, resource, extended) + }, true + } tbl, ok := lookupTable( node.OnTable.QualifierSecond.GetRawVal(), node.OnTable.Qualifier.GetRawVal(), diff --git a/internal/stackql/intrinsic/omnisdk.go b/internal/stackql/intrinsic/omnisdk.go index 2b5e4bb6..69e839cd 100644 --- a/internal/stackql/intrinsic/omnisdk.go +++ b/internal/stackql/intrinsic/omnisdk.go @@ -156,9 +156,10 @@ func openStream( } input := previewCfg args := omnisdk.Args{ - Params: params, - Auth: omnisdkAuth(providerAuthContext(ctx, resourcePath)), - Endpoint: input.getEndpoint(), + Params: params, + Auth: omnisdkAuth(providerAuthContext(ctx, resourcePath)), + Endpoint: input.getEndpoint(), + InsecureSkipTLSVerify: input.getInsecureSkipTLSVerify(), } plan, err := omnisdk.Default().New(method.Path, args) if err != nil { @@ -188,6 +189,7 @@ type rowStream struct { producerOnce sync.Once columns []column table sqldata.ISQLTable + projection sqlparser.SelectExprs typCfg columnFactory done bool } @@ -263,6 +265,11 @@ func (rs *rowStream) result(batch []omnisdk.Row) sqldata.ISQLResult { for _, name := range sortedKeys(batch[0]) { rs.columns = append(rs.columns, column{name: name}) } + if len(rs.projection) > 0 { + if selected, err := projection(rs.projection, rs.columns); err == nil { + rs.columns = selected + } + } } columns := make([]sqldata.ISQLColumn, 0, len(rs.columns)) for _, col := range rs.columns { @@ -312,6 +319,11 @@ func selectFunc( if !ok { return nil, false } + if bundle, isDoc := docProvider( + resolveProvider(tableName.QualifierSecond.GetRawVal(), currentProvider)); isDoc { + return docSelectFunc(ctx, node, bundle, + tableName.Qualifier.GetRawVal(), tableName.Name.GetRawVal()) + } if !strings.EqualFold(tableName.Qualifier.GetRawVal(), auditService) || !IsProvider(resolveProvider(tableName.QualifierSecond.GetRawVal(), currentProvider)) { return nil, false @@ -644,12 +656,14 @@ type backendInput interface { getBatchSize() int getEndpoint() string getFlushInterval() time.Duration + getInsecureSkipTLSVerify() bool } type standardBackendInput struct { - batchSize int - endpoint string - flushInterval time.Duration + batchSize int + endpoint string + flushInterval time.Duration + insecureSkipTLSVerify bool } // previewCfg is the parsed --preview argument. Cobra binds the raw string in @@ -665,9 +679,10 @@ const CfgRawKey = "preview" // either form omnisdk does: a base URL for every service, or an object of // service to override. Both ride through as the string omnisdk parses. type previewCfgDTO struct { - BatchSize int `json:"batchSize"` - FlushInterval string `json:"flushInterval"` - Endpoint json.RawMessage `json:"endpoint"` + BatchSize int `json:"batchSize"` + FlushInterval string `json:"flushInterval"` + Endpoint json.RawMessage `json:"endpoint"` + InsecureSkipTLSVerify bool `json:"insecureSkipTLSVerify"` } func (c previewCfgDTO) endpoint() string { @@ -694,9 +709,10 @@ func Init(raw string) { func newBackendInput(cfg previewCfgDTO) backendInput { rv := &standardBackendInput{ - batchSize: defaultBatchSize, - endpoint: cfg.endpoint(), - flushInterval: defaultFlushInterval, + batchSize: defaultBatchSize, + endpoint: cfg.endpoint(), + flushInterval: defaultFlushInterval, + insecureSkipTLSVerify: cfg.InsecureSkipTLSVerify, } if cfg.BatchSize > 0 { rv.batchSize = cfg.BatchSize @@ -713,6 +729,8 @@ func (b *standardBackendInput) getEndpoint() string { return b.endpoint } func (b *standardBackendInput) getFlushInterval() time.Duration { return b.flushInterval } +func (b *standardBackendInput) getInsecureSkipTLSVerify() bool { return b.insecureSkipTLSVerify } + // sourceKey is the row key a column reads from: its own name, unless an alias // renamed it. func (c column) sourceKey() string { diff --git a/test/assets/expected/preview/omni-iam-principals.jsonl b/test/assets/expected/preview/omni-iam-principals.jsonl new file mode 100644 index 00000000..8fb13465 --- /dev/null +++ b/test/assets/expected/preview/omni-iam-principals.jsonl @@ -0,0 +1,13 @@ +{"created": "2024-01-05T00:00:00Z", "enabled": "true", "google_org": "123456789", "google_project": null, "grant": "Global Reader", "mfa": null, "principal": "reviewer@example.invalid", "principal_id": "00000000-0000-0000-0000-000000000001", "principal_type": "user", "provider": "entra", "region": "us-east-1", "scope": "mock-tenant"} +{"created": "2024-01-05T00:00:00Z", "enabled": null, "google_org": "123456789", "google_project": null, "grant": "ReadOnlyAccess", "mfa": "true", "principal": "audit-reviewer", "principal_id": "AIDAEXAMPLEREVIEWER01", "principal_type": "user", "provider": "aws", "region": "us-east-1", "scope": "000000000000"} +{"created": "2024-02-11T00:00:00Z", "enabled": null, "google_org": "123456789", "google_project": null, "grant": "AmazonS3FullAccess", "mfa": "false", "principal": "ci-deployer", "principal_id": "AIDAEXAMPLEDEPLOYER02", "principal_type": "user", "provider": "aws", "region": "us-east-1", "scope": "000000000000"} +{"created": "2024-02-11T00:00:00Z", "enabled": null, "google_org": "123456789", "google_project": null, "grant": "IAMReadOnlyAccess", "mfa": "false", "principal": "ci-deployer", "principal_id": "AIDAEXAMPLEDEPLOYER02", "principal_type": "user", "provider": "aws", "region": "us-east-1", "scope": "000000000000"} +{"created": "2024-03-19T00:00:00Z", "enabled": null, "google_org": "123456789", "google_project": null, "grant": null, "mfa": "false", "principal": "no-grants-user", "principal_id": "AIDAEXAMPLENOGRANTS03", "principal_type": "user", "provider": "aws", "region": "us-east-1", "scope": "000000000000"} +{"created": "2024-04-02T00:00:00Z", "enabled": "false", "google_org": "123456789", "google_project": null, "grant": "Application Administrator", "mfa": null, "principal": "operator@example.invalid", "principal_id": "00000000-0000-0000-0000-000000000002", "principal_type": "user", "provider": "entra", "region": "us-east-1", "scope": "mock-tenant"} +{"created": "2024-04-02T00:00:00Z", "enabled": "false", "google_org": "123456789", "google_project": null, "grant": "Directory Readers", "mfa": null, "principal": "operator@example.invalid", "principal_id": "00000000-0000-0000-0000-000000000002", "principal_type": "user", "provider": "entra", "region": "us-east-1", "scope": "mock-tenant"} +{"created": null, "enabled": null, "google_org": "123456789", "google_project": null, "grant": "roles/editor", "mfa": null, "principal": "ci@example-project.iam.gserviceaccount.com", "principal_id": "serviceAccount:ci@example-project.iam.gserviceaccount.com", "principal_type": "service_principal", "provider": "gcp", "region": "us-east-1", "scope": "proj-100"} +{"created": null, "enabled": null, "google_org": "123456789", "google_project": null, "grant": "roles/editor", "mfa": null, "principal": "ci@example-project.iam.gserviceaccount.com", "principal_id": "serviceAccount:ci@example-project.iam.gserviceaccount.com", "principal_type": "service_principal", "provider": "gcp", "region": "us-east-1", "scope": "proj-root"} +{"created": null, "enabled": null, "google_org": "123456789", "google_project": null, "grant": "roles/editor", "mfa": null, "principal": "operator@example.invalid", "principal_id": "user:operator@example.invalid", "principal_type": "user", "provider": "gcp", "region": "us-east-1", "scope": "proj-100"} +{"created": null, "enabled": null, "google_org": "123456789", "google_project": null, "grant": "roles/editor", "mfa": null, "principal": "operator@example.invalid", "principal_id": "user:operator@example.invalid", "principal_type": "user", "provider": "gcp", "region": "us-east-1", "scope": "proj-root"} +{"created": null, "enabled": null, "google_org": "123456789", "google_project": null, "grant": "roles/viewer", "mfa": null, "principal": "reviewer@example.invalid", "principal_id": "user:reviewer@example.invalid", "principal_type": "user", "provider": "gcp", "region": "us-east-1", "scope": "proj-100"} +{"created": null, "enabled": null, "google_org": "123456789", "google_project": null, "grant": "roles/viewer", "mfa": null, "principal": "reviewer@example.invalid", "principal_id": "user:reviewer@example.invalid", "principal_type": "user", "provider": "gcp", "region": "us-east-1", "scope": "proj-root"} diff --git a/test/python/stackql_test_tooling/flask/omnisdk/app.py b/test/python/stackql_test_tooling/flask/omnisdk/app.py index 8ace9b15..92f4361f 100644 --- a/test/python/stackql_test_tooling/flask/omnisdk/app.py +++ b/test/python/stackql_test_tooling/flask/omnisdk/app.py @@ -28,6 +28,9 @@ app = Flask(__name__, template_folder=str(HERE / "templates")) BUCKETS = json.loads((COLL / "buckets.json").read_text()) PROVISION = json.loads((COLL / "ec2" / "provision.json").read_text()) +IAM_USERS = json.loads((COLL / "iam_users.json").read_text()) +ENTRA_USERS = json.loads((COLL / "entra_users.json").read_text()) +GCP_IAM_POLICY = json.loads((COLL / "gcp_iam_policy.json").read_text()) NOT_FOUND = ( '' @@ -64,6 +67,8 @@ def list_buckets(): def ec2_query(): # EC2 Query API: params ride the POST form body (Action, VpcId, ...). action = request.form.get("Action") + if action in ("ListUsers", "ListAttachedUserPolicies", "ListMFADevices"): + return _iam_query(action) if action == "CreateVpc": return Response(render_template("create_vpc.xml.j2", **PROVISION), mimetype=XML) if action == "CreateSubnet": @@ -74,6 +79,63 @@ def ec2_query(): return Response("InvalidAction", status=400, mimetype=XML) +def _iam_query(action): + # AWS IAM Query API. Left-outer by nature: a user with no policy or no MFA + # device still answers, with an empty member list. + if action == "ListUsers": + members = "".join( + f"{u['name']}{u['id']}" + f"{u['created']}" + f"arn:aws:iam::000000000000:user/{u['name']}" + for u in IAM_USERS + ) + body = (f"{members}" + "false") + return Response(body, mimetype=XML) + who = request.form.get("UserName") + user = next((u for u in IAM_USERS if u["name"] == who), None) + if action == "ListAttachedUserPolicies": + members = "".join( + f"{p}" + f"arn:aws:iam::aws:policy/{p}" + for p in (user or {}).get("policies", []) + ) + body = ("" + f"{members}" + "") + return Response(body, mimetype=XML) + members = "".join( + f"{m}{who}" + for m in (user or {}).get("mfa", []) + ) + body = ("" + f"{members}" + "") + return Response(body, mimetype=XML) + + +@app.get("/v1.0/users") +def entra_users(): + return jsonify({"value": [ + {k: u[k] for k in ("userPrincipalName", "id", "accountEnabled", "createdDateTime")} + for u in ENTRA_USERS + ]}) + + +@app.get("/v1.0/users//memberOf") +def entra_member_of(principal_id): + user = next((u for u in ENTRA_USERS if u["id"] == principal_id), None) + return jsonify({"value": [ + {"displayName": r, "@odata.type": "#microsoft.graph.directoryRole"} + for r in (user or {}).get("roles", []) + ]}) + + +@app.post("/v3/projects/:getIamPolicy") +def gcp_get_iam_policy(project): + return jsonify(GCP_IAM_POLICY) + + @app.get("/") def bucket_op(bucket): if _DELAY_SECONDS > 0: diff --git a/test/python/stackql_test_tooling/flask/omnisdk/collateral/entra_users.json b/test/python/stackql_test_tooling/flask/omnisdk/collateral/entra_users.json new file mode 100644 index 00000000..a391462d --- /dev/null +++ b/test/python/stackql_test_tooling/flask/omnisdk/collateral/entra_users.json @@ -0,0 +1,21 @@ +[ + { + "userPrincipalName": "reviewer@example.invalid", + "id": "00000000-0000-0000-0000-000000000001", + "accountEnabled": true, + "createdDateTime": "2024-01-05T00:00:00Z", + "roles": [ + "Global Reader" + ] + }, + { + "userPrincipalName": "operator@example.invalid", + "id": "00000000-0000-0000-0000-000000000002", + "accountEnabled": false, + "createdDateTime": "2024-04-02T00:00:00Z", + "roles": [ + "Directory Readers", + "Application Administrator" + ] + } +] \ No newline at end of file diff --git a/test/python/stackql_test_tooling/flask/omnisdk/collateral/gcp_iam_policy.json b/test/python/stackql_test_tooling/flask/omnisdk/collateral/gcp_iam_policy.json new file mode 100644 index 00000000..d0a68f04 --- /dev/null +++ b/test/python/stackql_test_tooling/flask/omnisdk/collateral/gcp_iam_policy.json @@ -0,0 +1,17 @@ +{ + "bindings": [ + { + "role": "roles/viewer", + "members": [ + "user:reviewer@example.invalid" + ] + }, + { + "role": "roles/editor", + "members": [ + "serviceAccount:ci@example-project.iam.gserviceaccount.com", + "user:operator@example.invalid" + ] + } + ] +} \ No newline at end of file diff --git a/test/python/stackql_test_tooling/flask/omnisdk/collateral/iam_users.json b/test/python/stackql_test_tooling/flask/omnisdk/collateral/iam_users.json new file mode 100644 index 00000000..e06887cd --- /dev/null +++ b/test/python/stackql_test_tooling/flask/omnisdk/collateral/iam_users.json @@ -0,0 +1,30 @@ +[ + { + "name": "audit-reviewer", + "id": "AIDAEXAMPLEREVIEWER01", + "created": "2024-01-05T00:00:00Z", + "policies": [ + "ReadOnlyAccess" + ], + "mfa": [ + "arn:aws:iam::000000000000:mfa/audit-reviewer" + ] + }, + { + "name": "ci-deployer", + "id": "AIDAEXAMPLEDEPLOYER02", + "created": "2024-02-11T00:00:00Z", + "policies": [ + "AmazonS3FullAccess", + "IAMReadOnlyAccess" + ], + "mfa": [] + }, + { + "name": "no-grants-user", + "id": "AIDAEXAMPLENOGRANTS03", + "created": "2024-03-19T00:00:00Z", + "policies": [], + "mfa": [] + } +] \ No newline at end of file diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index 26c912b5..a4eb87a3 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -7590,13 +7590,15 @@ Alternate App Root Persists All Temp Materials in Alotted Directory ... registry pull google v0.1.2; ... show providers; ${outputStr} = Catenate SEPARATOR=\n - ... |-----------------|----------| - ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}version${SPACE}${SPACE}| - ... |-----------------|----------| - ... |${SPACE}google${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}v0.1.2${SPACE}${SPACE}${SPACE}| - ... |-----------------|----------| - ... |${SPACE}stackql_preview${SPACE}|${SPACE}internal${SPACE}| - ... |-----------------|----------| + ... |-------------------------|----------| + ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}version${SPACE}${SPACE}| + ... |-------------------------|----------| + ... |${SPACE}google${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}v0.1.2${SPACE}${SPACE}${SPACE}| + ... |-------------------------|----------| + ... |${SPACE}stackql_preview${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}internal${SPACE}| + ... |-------------------------|----------| + ... |${SPACE}stackql_unstable_google${SPACE}|${SPACE}internal${SPACE}| + ... |-------------------------|----------| ${outputErrStr} = Catenate SEPARATOR=\n ... google provider, version 'v0.1.2' successfully installed Should Stackql Exec Inline Equal Both Streams @@ -10622,7 +10624,7 @@ Preview Show Resources Mirrors Omnisdk Catalog ... ${AUTH_CFG_STR} ... ${SQL_BACKEND_CFG_STR_CANONICAL} ... show resources in stackql_preview.audit; - ... name,id\naws_ec2_networks,stackql_preview.audit.aws_ec2_networks\naws_s3_buckets,stackql_preview.audit.aws_s3_buckets\nazure_network_subnets,stackql_preview.audit.azure_network_subnets\nazure_storage_containers,stackql_preview.audit.azure_storage_containers\ngcp_compute_networks,stackql_preview.audit.gcp_compute_networks\ngoogle_storage_buckets,stackql_preview.audit.google_storage_buckets\nomni_storage_buckets,stackql_preview.audit.omni_storage_buckets + ... name,id\naws_ec2_networks,stackql_preview.audit.aws_ec2_networks\naws_iam_principals,stackql_preview.audit.aws_iam_principals\naws_s3_buckets,stackql_preview.audit.aws_s3_buckets\nazure_network_subnets,stackql_preview.audit.azure_network_subnets\nazure_storage_containers,stackql_preview.audit.azure_storage_containers\nentra_identities,stackql_preview.audit.entra_identities\ngcp_compute_networks,stackql_preview.audit.gcp_compute_networks\ngcp_iam_principals,stackql_preview.audit.gcp_iam_principals\ngoogle_storage_buckets,stackql_preview.audit.google_storage_buckets\nomni_iam_principals,stackql_preview.audit.omni_iam_principals\nomni_storage_buckets,stackql_preview.audit.omni_storage_buckets ... \-o\=csv ... stdout=${CURDIR}${/}tmp${/}Preview-Show-Resources-Mirrors-Omnisdk-Catalog.tmp ... stderr=${CURDIR}${/}tmp${/}Preview-Show-Resources-Mirrors-Omnisdk-Catalog-stderr.tmp @@ -10725,12 +10727,16 @@ Preview Jsonl Row Set Is Order Insensitive ... emit, so an order-sensitive comparison would fail here. ${expected} = Catenate SEPARATOR=\n ... {"id":"stackql_preview.audit.omni_storage_buckets","name":"omni_storage_buckets"} - ... {"id":"stackql_preview.audit.aws_s3_buckets","name":"aws_s3_buckets"} - ... {"id":"stackql_preview.audit.gcp_compute_networks","name":"gcp_compute_networks"} - ... {"id":"stackql_preview.audit.aws_ec2_networks","name":"aws_ec2_networks"} + ... {"id":"stackql_preview.audit.omni_iam_principals","name":"omni_iam_principals"} ... {"id":"stackql_preview.audit.google_storage_buckets","name":"google_storage_buckets"} - ... {"id":"stackql_preview.audit.azure_network_subnets","name":"azure_network_subnets"} + ... {"id":"stackql_preview.audit.gcp_iam_principals","name":"gcp_iam_principals"} + ... {"id":"stackql_preview.audit.gcp_compute_networks","name":"gcp_compute_networks"} + ... {"id":"stackql_preview.audit.entra_identities","name":"entra_identities"} ... {"id":"stackql_preview.audit.azure_storage_containers","name":"azure_storage_containers"} + ... {"id":"stackql_preview.audit.azure_network_subnets","name":"azure_network_subnets"} + ... {"id":"stackql_preview.audit.aws_s3_buckets","name":"aws_s3_buckets"} + ... {"id":"stackql_preview.audit.aws_iam_principals","name":"aws_iam_principals"} + ... {"id":"stackql_preview.audit.aws_ec2_networks","name":"aws_ec2_networks"} Should StackQL Exec Inline Jsonl Set Equal ... ${STACKQL_EXE} ... ${OKTA_SECRET_STR} @@ -10779,7 +10785,7 @@ Preview Omni Storage Buckets Jsonl Row Set Matches Expectation ... ${OKTA_SECRET_STR} ... ${GITHUB_SECRET_STR} ... ${K8S_SECRET_STR} - ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${REGISTRY_MOCKED_CFG_STR} ... ${AUTH_CFG_STR} ... ${SQL_BACKEND_CFG_STR_CANONICAL} ... ${query} @@ -10840,3 +10846,126 @@ Preview Order By Is Refused Rather Than Ignored ... streams its rows, so ORDER BY cannot be applied; remove it from the query ... stdout=${CURDIR}${/}tmp${/}Preview-Order-By-Is-Refused-Rather-Than-Ignored.tmp ... stderr=${CURDIR}${/}tmp${/}Preview-Order-By-Is-Refused-Rather-Than-Ignored-stderr.tmp + +Preview Omni Iam Principals Jsonl Row Set Matches Expectation + [Documentation] The cross-cloud access review, order-insensitive. Every + ... identity source the fan-out touches is retargeted at the + ... mock: AWS IAM, the Entra login exchange and Graph, and the + ... GCP oauth exchange and CRM policy read. No ORDER BY, + ... because a streamed relation never reaches the SQL backend. + [Setup] Write Gcp Service Account ${OMNISDK_MOCK_GCP_SA_HOST} + [Teardown] Remove Preview Mock Environment + ${expected} = OperatingSystem.Get File + ... ${CURDIR}${/}..${/}..${/}assets${/}expected${/}preview${/}omni-iam-principals.jsonl + ${gcp_sa} = Set Variable If "${EXECUTION_PLATFORM}" == "docker" + ... /opt/test/tmp/omnisdk-gcp-sa.json ${OMNISDK_MOCK_GCP_SA_HOST} + Set Environment Variable AWS_ACCESS_KEY_ID AK + Set Environment Variable AWS_SECRET_ACCESS_KEY SK + Set Environment Variable AZURE_TENANT_ID mock-tenant + Set Environment Variable AZURE_CLIENT_ID mock-client + Set Environment Variable AZURE_CLIENT_SECRET mock-secret + Set Environment Variable GOOGLE_APPLICATION_CREDENTIALS ${gcp_sa} + ${mock} = Set Variable {"scheme":"http","host":"${LOCAL_HOST_ALIAS}","port":"${MOCKSERVER_PORT_OMNISDK}"} + ${preview} = Catenate SEPARATOR= + ... {"endpoint":{"aws.iam":${mock}, + ... "azure.login":${mock},"azure.graph":${mock}, + ... "gcp.oauth":${mock},"gcp.crm":${mock}}} + ${query} = Catenate SEPARATOR=${SPACE} + ... select * from stackql_preview.audit.omni_iam_principals + ... where region = 'us-east-1' and google_org = '123456789' and method = 'access'; + Should StackQL Exec Inline Jsonl Set Equal + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_MOCKED_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${query} + ... ${expected} + ... --preview\=${preview} + ... stdout=${CURDIR}${/}tmp${/}Preview-Omni-Iam-Principals-Jsonl-Row-Set-Matches-Expectation.tmp + ... stderr=${CURDIR}${/}tmp${/}Preview-Omni-Iam-Principals-Jsonl-Row-Set-Matches-Expectation-stderr.tmp + + + +Unstable Github Org Members Jsonl Row Set Matches Expectation + [Documentation] A document-derived SELECT: the provider document the + ... registry ships, run against the existing github mock. The + ... document declares no ordering, so the rows are asserted as + ... an unordered set, declared here in an order the query does + ... not emit. + [Teardown] Remove Preview Mock Environment + ${approot_gh} = Normalize Path ${REPOSITORY_ROOT}${/}test${/}registry-mocked-native + ${preview} = Catenate SEPARATOR= + ... {"endpoint":"https://${LOCAL_HOST_ALIAS}:${MOCKSERVER_PORT_GITHUB}", + ... "insecureSkipTLSVerify":true} + ${expected} = Catenate SEPARATOR=\n + ... {"id":"1","login":"some-jimbo-10","type":"User"} + ... {"id":"1","login":"some-jimbo-9","type":"User"} + ... {"id":"1","login":"some-jimbo-8","type":"User"} + ... {"id":"1","login":"some-jimbo-7","type":"User"} + ... {"id":"1","login":"some-jimbo-6","type":"User"} + ... {"id":"1","login":"some-jimbo-5","type":"User"} + ... {"id":"1","login":"some-jimbo-4","type":"User"} + ... {"id":"1","login":"some-jimbo-3","type":"User"} + ... {"id":"1","login":"some-jimbo-2","type":"User"} + ... {"id":"1","login":"some-jimbo-1","type":"User"} + ${query} = Catenate SEPARATOR=${SPACE} + ... select login, id, type from stackql_unstable_github.orgs.members + ... where org = 'dummyorg'; + Should StackQL Exec Inline Jsonl Set Equal + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_MOCKED_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${query} + ... ${expected} + ... --preview\=${preview} + ... stackql_approot=${approot_gh} + ... stdout=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members.tmp + ... stderr=${CURDIR}${/}tmp${/}Unstable-Github-Org-Members-stderr.tmp + +Unstable Google Storage Buckets Jsonl Row Set Matches Expectation + [Documentation] The document-derived counterpart of the registry-backed + ... `google.storage.buckets` listing, run against the same + ... google mock. The document declares no ordering, so rows + ... are asserted as an unordered set. + [Setup] Write Gcp Service Account ${OMNISDK_MOCK_GCP_SA_HOST} + [Teardown] Remove Preview Mock Environment + ${gcp_sa} = Set Variable If "${EXECUTION_PLATFORM}" == "docker" + ... /opt/test/tmp/omnisdk-gcp-sa.json ${OMNISDK_MOCK_GCP_SA_HOST} + Set Environment Variable GOOGLE_APPLICATION_CREDENTIALS ${gcp_sa} + ${preview} = Catenate SEPARATOR= + ... {"endpoint":"https://${LOCAL_HOST_ALIAS}:${MOCKSERVER_PORT_GOOGLE}", + ... "insecureSkipTLSVerify":true} + # The suite's google credentials are PKCS1; omnisdk needs PKCS8, so this + # points at the key the setup generates. + ${auth} = Set Variable {"google":{"credentialsfilepath":"${gcp_sa}"}} + # A private copy: a shared app root is written to by other tests, and which + # google bundle is on disk decides whether this resource has a select verb. + ${approot} = Normalize Path ${CURDIR}${/}tmp${/}unstable-google-approot + Remove Directory ${approot} recursive=True + Copy Directory + ... ${REPOSITORY_ROOT}${/}test${/}registry-mocked-native${/}src${/}googleapis.com + ... ${approot}${/}src${/}googleapis.com + ${query} = Catenate SEPARATOR=${SPACE} + ... select kind, project from stackql_unstable_google.storage.buckets + ... where project = 'stackql-demo'; + Should StackQL Exec Inline Jsonl Set Equal + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_MOCKED_CFG_STR} + ... ${auth} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${query} + ... {"kind":"storage#buckets","project":"stackql-demo"} + ... --preview\=${preview} + ... stackql_approot=${approot} + ... stdout=${CURDIR}${/}tmp${/}Unstable-Google-Storage-Buckets.tmp + ... stderr=${CURDIR}${/}tmp${/}Unstable-Google-Storage-Buckets-stderr.tmp