Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .github/skills/generate-api-markdown/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ description: Generate an API markdown file and token file using ApiView. Use thi
1. Navigate to the desired package directory
2. Run the command:
```bash
azpysdk apistub --md --extract-metadata --install-deps --dest-dir . .
azpysdk apistub .
```
3. The command outputs the location of the generated markdown file. Provide this file to the user for review.
3. The command generates `api.md` and `api.metadata.yml` in the package directory, which are the files needed to pass the API consistency check. Provide these files to the user for review.
4. If the user explicitly asks for the raw APIView token file, run `azpysdk apistub . --token-file` instead.
4 changes: 2 additions & 2 deletions .github/workflows/src/api-md-consistency/adapters/python.js
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ function generateApiForPackage({
const pythonExecutable = runtimeExecutable || process.env.RUNTIME_EXECUTABLE;
run(
pythonExecutable,
["-m", "azpysdk.main", "apistub", "--md", "--extract-metadata", "--dest-dir", packageDir, packageName],
["-m", "azpysdk.main", "apistub", packageName],
{
cwd: repoRoot,
check: true,
Expand All @@ -141,7 +141,7 @@ function generateApiForPackage({
return;
}

run("azpysdk", ["apistub", "--md", "--extract-metadata", "--dest-dir", packageDir, packageName], {
run("azpysdk", ["apistub", packageName], {
cwd: repoRoot,
check: true,
logger: activeLogger,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ function formatIssueSection(title, apiFiles) {
lines.push(styleLog(`PACKAGE: ${packageName}`, ANSI.bold, ANSI.cyan));
lines.push(`PATH: ${packageDir}`);
lines.push(`API FILE: ${apiFile}`);
lines.push(styleLog("Regenerate from the repository root:", ANSI.bold, ANSI.yellow));
lines.push(styleLog(` azpysdk apistub --md --extract-metadata ${packageName} --dest-dir .`, ANSI.bold, ANSI.yellow));
lines.push(styleLog(`Regenerate from the ${packageName} package root:`, ANSI.bold, ANSI.yellow));
lines.push(styleLog(` azpysdk apistub .`, ANSI.bold, ANSI.yellow));
lines.push("============================================================");
}
lines.push("");
Expand Down Expand Up @@ -97,7 +97,7 @@ export default async function apiMdConsistency({ core }) {
"",
formatIssueSection("Mismatched packages:", mismatches),
formatIssueSection("Missing required API files:", missing),
"To regenerate api.md locally, run the command shown for each package from the repository root.",
"To regenerate api.md and api.metadata.yml locally, run the command shown for each package from the repository root.",
].filter((part) => part !== "");

core.setFailed(messageParts.join("\n"));
Expand Down
2 changes: 1 addition & 1 deletion doc/dev/tests.md
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ A quick description of the commands above:
- verifywhl: verifies the wheel contents and manifest
- verifysdist: verifies the sdist contents and manifest
- samples: runs all of the samples in the `samples` directory and verifies they are working correctly
- apistub: runs the [apistubgenerator](https://github.com/Azure/azure-sdk-tools/tree/main/packages/python-packages/apiview-stub-generator) tool on your code
- apistub: runs the [apistubgenerator](https://github.com/Azure/azure-sdk-tools/tree/main/packages/python-packages/apiview-stub-generator) tool on your code. By default, generates `api.md` and `api.metadata.yml` in the package directory, which are the files needed to pass the API consistency check. Use `--token-file` to generate only the raw APIView token file.

## The `devtools_testutils` package

Expand Down
2 changes: 1 addition & 1 deletion doc/tool_usage_guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ The following checks are available via the `azpysdk` entrypoint.
|`pyright`| Runs `pyright` checks or `next-pyright` checks. (based on presence of `--next` argument) | `azpysdk pyright .` |
|`black`| Runs `black` checks. | `azpysdk black .` |
|`verifytypes`| Runs `verifytypes` checks. | `azpysdk verifytypes .` |
|`apistub`| Generates an api stub for the package. | `azpysdk apistub .` |
|`apistub`| Generates `api.md`, API metadata, or APIView token files for the package. | `azpysdk apistub .` |
|`bandit`| Runs `bandit` checks, which detect common security issues. | `azpysdk bandit .` |
|`verifywhl`| Verifies that the root directory in whl is azure, and verifies manifest so that all directories in source are included in sdist. | `azpysdk verifywhl .` |
|`verifysdist`| Verify directories included in sdist and contents in manifest file. Also ensures that py.typed configuration is correct within the setup.py. | `azpysdk verifysdist .` |
Expand Down
63 changes: 25 additions & 38 deletions eng/tools/azure-sdk-tools/azpysdk/apistub.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,24 +57,11 @@ def register(
"apistub", parents=parents, help="Run the apistub check to generate an API stub for a package"
)
p.add_argument(
"--dest-dir",
dest="dest_dir",
default=None,
help="Destination directory for generated API stub token files.",
)
p.add_argument(
"--md",
dest="generate_md",
default=False,
action="store_true",
help="Generate api.md from the JSON token file using Export-APIViewMarkdown.ps1. Output directory for api.md is the same as the generated token file.",
)
p.add_argument(
"--extract-metadata",
dest="extract_metadata",
"--token-file",
dest="token_file",
default=False,
action="store_true",
help="Extract language-specific metadata from generated api.md into api.metadata.yml and remove metadata header from api.md.",
help="Generate only the raw APIView token file.",
)
Comment thread
tjprescott marked this conversation as resolved.
p.add_argument(
"--install-deps",
Expand Down Expand Up @@ -106,9 +93,8 @@ def run(self, args: argparse.Namespace) -> int:
"""Run the apistub check command."""
logger.info("Running apistub check...")

if getattr(args, "extract_metadata", False) and not getattr(args, "generate_md", False):
logger.error("--extract-metadata requires --md.")
return 1
token_file = getattr(args, "token_file", False)
generate_markdown = not token_file

set_envvar_defaults()
targeted = self.get_targeted_directories(args)
Expand Down Expand Up @@ -160,12 +146,8 @@ def run(self, args: argparse.Namespace) -> int:
pkg_path = get_package_wheel_path(package_dir)
pkg_path = os.path.abspath(pkg_path)

dest_dir = getattr(args, "dest_dir", None)
if dest_dir:
out_token_path = os.path.abspath(dest_dir)
os.makedirs(out_token_path, exist_ok=True)
else:
out_token_path = os.path.abspath(staging_directory)
out_token_path = os.path.abspath(package_dir)
os.makedirs(out_token_path, exist_ok=True)

cross_language_mapping_path = get_cross_language_mapping_path(package_dir)

Expand All @@ -178,15 +160,21 @@ def run(self, args: argparse.Namespace) -> int:
cmds.extend(["--out-path", out_token_path])
if cross_language_mapping_path:
cmds.extend(["--mapping-path", cross_language_mapping_path])
if getattr(args, "generate_md", False):
if generate_markdown:
cmds.append("--skip-pylint")

logger.info("Running apistub {}.".format(cmds))

try:
self.run_venv_command(executable, cmds, cwd=staging_directory, check=True, immediately_dump=True)
if getattr(args, "generate_md", False):
token_json_path = os.path.join(out_token_path, f"{package_name}_python.json")
token_json_path = os.path.join(out_token_path, f"{package_name}_python.json")
if token_file:
if os.path.exists(token_json_path):
logger.info(f"Generated APIView token file: {token_json_path}")
else:
logger.error(f"Expected APIView token file was not generated: {token_json_path}")
results.append(1)
else:
md_script = os.path.join(REPO_ROOT, "eng", "common", "scripts", "Export-APIViewMarkdown.ps1")
metadata_script = os.path.join(REPO_ROOT, "eng", "scripts", "Extract-APIViewMetadata-Python.ps1")
logger.info(f"Generating api.md for {package_name}")
Expand All @@ -201,16 +189,15 @@ def run(self, args: argparse.Namespace) -> int:
if result.stdout:
logger.info(result.stdout)

if getattr(args, "extract_metadata", False):
logger.info(f"Extracting API metadata for {package_name}")
metadata_result = run(
["pwsh", metadata_script, "-OutputPath", out_token_path],
check=True,
capture_output=True,
text=True,
)
if metadata_result.stdout:
logger.info(metadata_result.stdout)
logger.info(f"Extracting API metadata for {package_name}")
metadata_result = run(
["pwsh", metadata_script, "-OutputPath", out_token_path],
check=True,
capture_output=True,
text=True,
)
if metadata_result.stdout:
logger.info(metadata_result.stdout)
except FileNotFoundError:
logger.error("Failed to generate api.md: pwsh (PowerShell) is not installed or not on PATH.")
results.append(1)
Expand Down
12 changes: 8 additions & 4 deletions eng/tools/azure-sdk-tools/ci_tools/functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,15 +182,19 @@ def glob_packages(glob_string: str, target_root_dir: str) -> List[str]:
collected_top_level_directories = []

for glob_string in individual_globs:
globbed = glob.glob(os.path.join(target_root_dir, glob_string, "setup.py"), recursive=True) + glob.glob(
os.path.join(target_root_dir, "sdk/*/", glob_string, "setup.py")
globbed = (
glob.glob(os.path.join(target_root_dir, glob_string, "setup.py"), recursive=True)
+ glob.glob(os.path.join(target_root_dir, "*/", glob_string, "setup.py"))
+ glob.glob(os.path.join(target_root_dir, "sdk/*/", glob_string, "setup.py"))
)
collected_top_level_directories.extend([os.path.dirname(p) for p in globbed])

# handle pyproject.toml separately, as we need to filter them by the presence of a `[project]` section
for glob_string in individual_globs:
globbed = glob.glob(os.path.join(target_root_dir, glob_string, "pyproject.toml"), recursive=True) + glob.glob(
os.path.join(target_root_dir, "sdk/*/", glob_string, "pyproject.toml")
globbed = (
glob.glob(os.path.join(target_root_dir, glob_string, "pyproject.toml"), recursive=True)
+ glob.glob(os.path.join(target_root_dir, "*/", glob_string, "pyproject.toml"))
+ glob.glob(os.path.join(target_root_dir, "sdk/*/", glob_string, "pyproject.toml"))
)
for p in globbed:
if get_pyproject(os.path.dirname(p)):
Expand Down
4 changes: 0 additions & 4 deletions eng/tools/azure-sdk-tools/packaging_tools/sdk_generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -335,11 +335,7 @@ def main(generate_input, generate_output):
cmds = [
"azpysdk",
"apistub",
"--md",
"--extract-metadata",
package_name,
"--dest-dir",
package_path.absolute().as_posix(),
]
_LOGGER.info(f"generate apiview file for package {package_name}")
check_call(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
from ci_tools.parsing import ParsedSetup
from ci_tools.functions import discover_targeted_packages


repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", ".."))
sdk_root = os.path.join(repo_root, "sdk")
core_service_root = os.path.join(sdk_root, "core")
Expand Down Expand Up @@ -70,6 +69,22 @@ def test_discovery_single_package():
]


def test_discovery_single_package_from_sdk_root():
results = discover_targeted_packages("azure-template", sdk_root, filter_type="Build")

assert [os.path.basename(result) for result in results] == [
"azure-template",
]


def test_discovery_single_package_from_repo_root():
results = discover_targeted_packages("azure-template", repo_root, filter_type="Build")

assert [os.path.basename(result) for result in results] == [
"azure-template",
]


def test_discovery_omit_regression():
results = discover_targeted_packages("*", core_service_root, filter_type="Regression")

Expand Down
Loading
Loading