diff --git a/blog/posts/2024-03-14-commitment-to-opentelemetry.md b/blog/posts/2024-03-14-commitment-to-opentelemetry.md index 3cb1196d4..3ceb2c2c7 100644 --- a/blog/posts/2024-03-14-commitment-to-opentelemetry.md +++ b/blog/posts/2024-03-14-commitment-to-opentelemetry.md @@ -27,7 +27,7 @@ In [Prometheus v2.47.0](https://github.com/prometheus/prometheus/releases/tag/v2 ### Support UTF-8 metric and label names -[OpenTelemetry semantic conventions](https://github.com/open-telemetry/semantic-conventions/blob/main/docs/http/http-metrics.md) push for `“.”` to be the namespacing character. For example, `http.server.request.duration`. However, Prometheus currently requires a [more limited character set](https://prometheus.io/docs/instrumenting/writing_exporters/#naming), which means we convert the metric to `http_server_request_duration` when ingesting it into Prometheus. +[OpenTelemetry semantic conventions](https://github.com/open-telemetry/semantic-conventions/blob/main/docs/http/http-metrics.md) push for `“.”` to be the namespacing character. For example, `http.server.request.duration`. However, Prometheus currently requires a [more limited character set](https://prometheus.io/docs/exporters/writing-exporters/#naming), which means we convert the metric to `http_server_request_duration` when ingesting it into Prometheus. This causes unnecessary dissonance and we’re working on removing this limitation by adding UTF-8 support for all labels and metric names. The progress is tracked [here](https://github.com/prometheus/prometheus/issues/13095). diff --git a/docs-config.ts b/docs-config.ts index fb065179a..639fd2b6b 100644 --- a/docs-config.ts +++ b/docs-config.ts @@ -33,6 +33,7 @@ export default { repo: "prometheus", repoDocsDir: "docs", slugPrefix: "prometheus", + versioning: "release-branches", minNumVersions: 10, }, { @@ -40,8 +41,16 @@ export default { repo: "alertmanager", repoDocsDir: "docs", slugPrefix: "alerting", + versioning: "release-branches", minNumVersions: 8, }, + { + owner: "prometheus-community", + repo: "postgres_exporter", + repoDocsDir: "docs", + slugPrefix: "exporters/postgres", + versioning: "latest-release-tag", + }, ], // Single pages to fetch from external repos (not versioned). diff --git a/docs/instrumenting/exporters.md b/docs/exporters/exporters-and-integrations.md similarity index 98% rename from docs/instrumenting/exporters.md rename to docs/exporters/exporters-and-integrations.md index 1b78645a1..fc2e7ad1f 100644 --- a/docs/instrumenting/exporters.md +++ b/docs/exporters/exporters-and-integrations.md @@ -1,6 +1,6 @@ --- title: Exporters and integrations -sort_rank: 4 +sort_rank: 1 --- There are a number of libraries and servers which help in exporting existing @@ -14,7 +14,7 @@ Some of these exporters are maintained as part of the official [Prometheus GitHu those are marked as *official*, others are externally contributed and maintained. We encourage the creation of more exporters but cannot vet all of them for -[best practices](/docs/instrumenting/writing_exporters/). +[best practices](/docs/exporters/writing-exporters/). Commonly, those exporters are hosted outside of the Prometheus GitHub organization. @@ -50,7 +50,7 @@ wide variety of JVM-based applications, for example [Kafka](http://kafka.apache. * [OpenTSDB Exporter](https://github.com/cloudflare/opentsdb_exporter) * [Oracle DB Exporter](https://github.com/iamseth/oracledb_exporter) * [PgBouncer exporter](https://github.com/prometheus-community/pgbouncer_exporter) - * [PostgreSQL exporter](https://github.com/prometheus-community/postgres_exporter) + * [PostgreSQL exporter](/docs/exporters/postgres/) ([source](https://github.com/prometheus-community/postgres_exporter)) * [Presto exporter](https://github.com/yahoojapan/presto_exporter) * [ProxySQL exporter](https://github.com/percona/proxysql_exporter) * [RavenDB exporter](https://github.com/marcinbudny/ravendb_exporter) @@ -290,7 +290,7 @@ wide variety of JVM-based applications, for example [Kafka](http://kafka.apache. When implementing a new Prometheus exporter, please follow the -[guidelines on writing exporters](/docs/instrumenting/writing_exporters) +[guidelines on writing exporters](/docs/exporters/writing-exporters/) Please also consider consulting the [development mailing list](https://groups.google.com/forum/#!forum/prometheus-developers). We are happy to give advice on how to make your exporter as useful and consistent as diff --git a/docs/exporters/index.md b/docs/exporters/index.md new file mode 100644 index 000000000..cdf3bc65d --- /dev/null +++ b/docs/exporters/index.md @@ -0,0 +1,19 @@ +--- +title: Exporters +sort_rank: 6 +nav_icon: package +--- + +Exporters expose metrics from systems that cannot be instrumented directly with +Prometheus client libraries. The exporter documentation in this section is +maintained in each exporter's own repository and pulled into this website from +the latest stable release. + +The broader catalog of available exporters and integrations is listed in +[Exporters and integrations](/docs/exporters/exporters-and-integrations/). + +## Documented exporters + +* [PostgreSQL exporter](/docs/exporters/postgres/) (coming from + [prometheus-community/postgres_exporter](https://github.com/prometheus-community/postgres_exporter)) + diff --git a/docs/instrumenting/writing_exporters.md b/docs/exporters/writing-exporters.md similarity index 99% rename from docs/instrumenting/writing_exporters.md rename to docs/exporters/writing-exporters.md index a9b4223df..984bc1d91 100644 --- a/docs/instrumenting/writing_exporters.md +++ b/docs/exporters/writing-exporters.md @@ -1,6 +1,6 @@ --- title: Writing exporters -sort_rank: 5 +sort_rank: 2 --- If you are instrumenting your own code, the [general rules of how to @@ -529,5 +529,5 @@ port allocations. Once you’re ready to announce your exporter to the world, email the mailing list and send a PR to add it to [the list of available -exporters](/docs/instrumenting/exporters/) by editing [this GitHub -repository file](https://github.com/prometheus/docs/blob/main/docs/instrumenting/exporters.md). +exporters](/docs/exporters/exporters-and-integrations/) by editing [this GitHub +repository file](https://github.com/prometheus/docs/blob/main/docs/exporters/exporters-and-integrations.md). diff --git a/docs/guides/multi-target-exporter.md b/docs/guides/multi-target-exporter.md index f175e1a40..44ac9ef28 100644 --- a/docs/guides/multi-target-exporter.md +++ b/docs/guides/multi-target-exporter.md @@ -12,7 +12,7 @@ This guide will introduce you to the multi-target exporter pattern. To achieve t ## The multi-target exporter pattern? -By multi-target [exporter](/docs/instrumenting/exporters/) pattern we refer to a specific design, in which: +By multi-target [exporter](/docs/exporters/exporters-and-integrations/) pattern we refer to a specific design, in which: * the exporter will get the target’s metrics via a network protocol. * the exporter does not have to run on the machine the metrics are taken from. diff --git a/docs/introduction/faq.md b/docs/introduction/faq.md index 3806d3c70..a6b95cef7 100644 --- a/docs/introduction/faq.md +++ b/docs/introduction/faq.md @@ -172,7 +172,7 @@ jobs. ### What applications can Prometheus monitor out of the box? -See [the list of exporters and integrations](/docs/instrumenting/exporters/). +See [the list of exporters and integrations](/docs/exporters/exporters-and-integrations/). ### Can I monitor JVM applications via JMX? diff --git a/docs/introduction/overview.md b/docs/introduction/overview.md index a5b141bcf..b8bf5153a 100644 --- a/docs/introduction/overview.md +++ b/docs/introduction/overview.md @@ -47,7 +47,7 @@ optional: * the main [Prometheus server](https://github.com/prometheus/prometheus) which scrapes and stores time series data * [client libraries](/docs/instrumenting/clientlibs/) for instrumenting application code * a [push gateway](https://github.com/prometheus/pushgateway) for supporting short-lived jobs -* special-purpose [exporters](/docs/instrumenting/exporters/) for services like HAProxy, StatsD, Graphite, etc. +* special-purpose [exporters](/docs/exporters/exporters-and-integrations/) for services like HAProxy, StatsD, Graphite, etc. * an [alertmanager](https://github.com/prometheus/alertmanager) to handle alerts * various support tools diff --git a/docs/operating/integrations.md b/docs/operating/integrations.md index 438338163..64da364ee 100644 --- a/docs/operating/integrations.md +++ b/docs/operating/integrations.md @@ -4,7 +4,7 @@ sort_rank: 5 --- In addition to [client libraries](/docs/instrumenting/clientlibs/) and -[exporters and related libraries](/docs/instrumenting/exporters/), there are +[exporters and related libraries](/docs/exporters/exporters-and-integrations/), there are numerous other generic integration points in Prometheus. This page lists some of the integrations with these. diff --git a/docs/specs/prw/remote_write_spec_2_0.md b/docs/specs/prw/remote_write_spec_2_0.md index abca75ec3..175382cf6 100644 --- a/docs/specs/prw/remote_write_spec_2_0.md +++ b/docs/specs/prw/remote_write_spec_2_0.md @@ -437,7 +437,7 @@ Typically, Senders can detect when a time series will no longer be appended usin #### Metadata -Metadata SHOULD follow the official Prometheus guidelines for [Type](https://prometheus.io/docs/instrumenting/writing_exporters/#types) and [Help](https://prometheus.io/docs/instrumenting/writing_exporters/#help-strings). +Metadata SHOULD follow the official Prometheus guidelines for [Type](https://prometheus.io/docs/exporters/writing-exporters/#types) and [Help](https://prometheus.io/docs/exporters/writing-exporters/#help-strings). Metadata MAY follow the official OpenMetrics guidelines for [Unit](https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#unit). diff --git a/public/_redirects b/public/_redirects index 88a646207..81255399c 100644 --- a/public/_redirects +++ b/public/_redirects @@ -32,6 +32,10 @@ https://openmetrics.io/* https://prometheus.io/docs/specs/om/open_metrics_spec 3 # Redirect for HTTP SD docs, which briefly lived in the wrong category / repo. /docs/instrumenting/http_sd/ /docs/prometheus/latest/http_sd/ +# Moved exporter docs into their own section. +/docs/instrumenting/exporters/ /docs/exporters/exporters-and-integrations/ 302! +/docs/instrumenting/writing_exporters/ /docs/exporters/writing-exporters/ 302! + # Redirect for "disabled_features", which is now called "feature_flags". /docs/prometheus/latest/disabled_features/ /docs/prometheus/latest/feature_flags/ diff --git a/scripts/fetch-repo-docs.ts b/scripts/fetch-repo-docs.ts index b8e98c2b6..c4d3371e0 100644 --- a/scripts/fetch-repo-docs.ts +++ b/scripts/fetch-repo-docs.ts @@ -20,6 +20,14 @@ const OUTDIR = "./generated"; const docsCollection: DocsCollection = {}; const allRepoVersions: AllRepoVersions = {}; +const normalizeMarkdownFilePath = (filePath: string) => + filePath === "README.md" + ? "index.md" + : filePath.replace(/\/README\.md$/, "/index.md"); + +const markdownSlugPath = (filePath: string) => + normalizeMarkdownFilePath(filePath).replace(/(^|\/)index\.md$/, "").replace(/\.md$/, ""); + // Find all files (.md and others) recursively in a directory. const findFiles = (dir: string): string[] => { let results: string[] = []; @@ -44,27 +52,28 @@ const syncRepo = (owner: string, repo: string, repoDir: string) => { `git clone --bare --filter=blob:none https://github.com/${owner}/${repo}.git ${repoDir}` ); execSync(`git -C ${repoDir} config core.sparseCheckout true`); - } else { - execSync(`git -C ${repoDir} fetch --prune --quiet`); } + execSync( + `git -C ${repoDir} fetch origin '+refs/heads/*:refs/remotes/origin/*' '+refs/tags/*:refs/tags/*' --prune --quiet` + ); }; const checkoutVersionDocs = ( owner: string, repo: string, repoDir: string, - version: string, + checkoutName: string, + gitRef: string, workingTreeBase: string, repoDocsDir: string ) => { const workingTree = path.resolve( - `${workingTreeBase}/${owner}/${repo}/${version}` + `${workingTreeBase}/${owner}/${repo}/${checkoutName}` ); - const checkoutConfig = `${repoDir}/worktrees/${version}/info/sparse-checkout`; - const branch = `release-${version}`; + const checkoutConfig = `${repoDir}/worktrees/${checkoutName}/info/sparse-checkout`; console.log( - `Checking out ${branch} of ${owner}/${repo} from ${repoDir} into ${workingTree}...` + `Checking out ${gitRef} of ${owner}/${repo} from ${repoDir} into ${workingTree}...` ); if (!fs.existsSync(checkoutConfig) || !fs.existsSync(workingTree)) { @@ -75,7 +84,7 @@ const checkoutVersionDocs = ( execSync(`rm -rf ${workingTree}`); } execSync( - `cd ${repoDir} && git worktree prune && git worktree add --no-checkout ${workingTree} ${branch}` + `cd ${repoDir} && git worktree prune && git worktree add --no-checkout ${workingTree} ${gitRef}` ); if (!fs.existsSync(path.dirname(checkoutConfig))) { fs.mkdirSync(path.dirname(checkoutConfig), { recursive: true }); @@ -89,13 +98,100 @@ const checkoutVersionDocs = ( execSync(`git -C ${workingTree} clean --force`); }; -const fetchRepoDocs = async ({ +type IntegrateRepoDocsOptions = { + owner: string; + repo: string; + docsDir: string; + repoDocsDir: string; + assetsRoot: string; + includeFile: (filePath: string) => boolean; + addMarkdownDoc: (args: { + file: string; + filePath: string; + slugPath: string; + title: string; + navTitle?: string; + sortRank: number; + hideInNav?: boolean; + }) => void; +}; + +const integrateRepoDocs = ({ + owner, + repo, + docsDir, + repoDocsDir, + assetsRoot, + includeFile, + addMarkdownDoc, +}: IntegrateRepoDocsOptions) => { + const files = findFiles(docsDir).sort((a, b) => { + const aPath = path.relative(docsDir, a); + const bPath = path.relative(docsDir, b); + return markdownSlugPath(aPath).localeCompare(markdownSlugPath(bPath)); + }); + + for (const file of files) { + const filePath = path.relative(docsDir, file); + const normalizedFilePath = normalizeMarkdownFilePath(filePath); + + if (!includeFile(normalizedFilePath)) { + continue; + } + + if (file.endsWith(".md")) { + console.log("Found Markdown file:", filePath); + + const { + data: { + title, + nav_title: navTitle, + sort_rank: sortRank, + hide_in_nav: hideInNav, + }, + } = matter(fs.readFileSync(file, "utf-8")); + + if (!title) { + throw new Error(`Missing title in ${file}`); + } + if (sortRank == null) { + // Docs in https://github.com/prometheus/prometheus/tree/main/docs/command-line + // are currently missing sort_rank 😤 + if (!filePath.includes("command-line")) { + throw new Error(`Missing sort_rank in ${file}`); + } + } + + addMarkdownDoc({ + file, + filePath: path.posix.join( + repoDocsDir, + filePath.split(path.sep).join(path.posix.sep) + ), + slugPath: markdownSlugPath(filePath), + title, + navTitle, + sortRank: sortRank ?? 0, + hideInNav, + }); + } else { + console.log("Found non-Markdown asset file:", filePath); + const destDir = `${OUTDIR}/${assetsRoot}/${path.dirname(filePath)}`; + if (!fs.existsSync(destDir)) { + fs.mkdirSync(destDir, { recursive: true }); + } + fs.copyFileSync(file, `${destDir}/${path.basename(filePath)}`); + } + } +}; + +const fetchVersionedRepoDocs = async ({ owner, repo, repoDocsDir, minNumVersions, slugPrefix, -}: GithubMarkdownSource) => { +}: Extract) => { console.log(`Fetching releases and repo docs for ${owner}/${repo}...`); // Clone a bare repo with sparse checkout so we can get the docs at specific @@ -184,6 +280,7 @@ const fetchRepoDocs = async ({ repo, repoCheckoutDir, version, + `origin/release-${version}`, `${OUTDIR}/repo-docs`, repoDocsDir ); @@ -192,47 +289,33 @@ const fetchRepoDocs = async ({ // Store metadata about Markdown page files, copy non-Markdown // assets to the docs assets directory. const assetsRoot = `/repo-docs-assets/${owner}/${repo}/${version}`; - for (const file of findFiles(docsDir)) { - const filePath = path.relative(docsDir, file); - - if ( - owner === "prometheus" && - ["prometheus", "alertmanager"].includes(repo) && - filePath === "index.md" - ) { - // Skip the index.md file in the external repo, as it is not a real or conformant page. - console.log("Skipping Prometheus index.md file:", filePath); - continue; - } - - if (file.endsWith(".md")) { - console.log("Found Markdown file:", filePath); - - const { - data: { - title, - nav_title: navTitle, - sort_rank: sortRank, - hide_in_nav: hideInNav, - }, - } = matter(fs.readFileSync(file, "utf-8")); - - if (!title) { - throw new Error(`Missing title in ${file}`); - } - if (!sortRank) { - // Docs in https://github.com/prometheus/prometheus/tree/main/docs/command-line - // are currently missing sort_rank 😤 - if (!filePath.includes("command-line")) { - throw new Error(`Missing sort_rank in ${file}`); - } + integrateRepoDocs({ + owner, + repo, + docsDir, + repoDocsDir, + assetsRoot, + includeFile: (filePath) => { + if ( + owner === "prometheus" && + ["prometheus", "alertmanager"].includes(repo) && + filePath === "index.md" + ) { + // Skip the index.md file in the external repo, as it is not a real or conformant page. + console.log("Skipping Prometheus index.md file:", filePath); + return false; } - - const slug = path.join( - slugPrefix, - version, - filePath.replace(/(\/index)*\.md$/, "") - ); + return true; + }, + addMarkdownDoc: ({ + file, + slugPath, + title, + navTitle, + sortRank, + hideInNav, + }) => { + const slug = path.join(slugPrefix, version, slugPath); const newDoc: DocMetadata = { type: "repo-doc", slug, @@ -254,25 +337,114 @@ const fetchRepoDocs = async ({ docsCollection[slug] = newDoc; if (version === latestVersion) { - const latestSlug = path.join( - slugPrefix, - "latest", - filePath.replace(/(\/index)*\.md$/, "") - ); + const latestSlug = path.join(slugPrefix, "latest", slugPath); // Also add the latest version to the collection with // "latest" as the version in the slug. docsCollection[latestSlug] = { ...newDoc, slug: latestSlug }; } - } else { - console.log("Found non-Markdown asset file:", filePath); - const destDir = `${OUTDIR}/${assetsRoot}/${path.dirname(filePath)}`; - if (!fs.existsSync(destDir)) { - fs.mkdirSync(destDir, { recursive: true }); - } - fs.copyFileSync(file, `${destDir}/${path.basename(filePath)}`); - } + }, + }); + } +}; + +const getLatestReleaseTag = async (owner: string, repo: string) => { + const iterator = octokit.paginate.iterator(octokit.rest.repos.listReleases, { + owner, + repo, + per_page: 100, + }); + + for await (const { data: releases } of iterator) { + const latestRelease = releases.find( + (release) => !release.draft && !release.prerelease + ); + if (latestRelease) { + return latestRelease.tag_name; } } + + throw new Error(`No stable release found for ${owner}/${repo}.`); +}; + +const fetchLatestReleaseTagRepoDocs = async ({ + owner, + repo, + repoDocsDir, + slugPrefix, +}: Extract) => { + console.log(`Fetching latest release docs for ${owner}/${repo}...`); + + const repoCheckoutDir = `${OUTDIR}/repos/${owner}/${repo}.git`; + syncRepo(owner, repo, repoCheckoutDir); + + const gitRef = await getLatestReleaseTag(owner, repo); + checkoutVersionDocs( + owner, + repo, + repoCheckoutDir, + gitRef, + gitRef, + `${OUTDIR}/repo-docs`, + repoDocsDir + ); + + const docsDir = `${OUTDIR}/repo-docs/${owner}/${repo}/${gitRef}/${repoDocsDir}`; + const assetsRoot = `/repo-docs-assets/${owner}/${repo}/${gitRef}`; + + if (!fs.existsSync(docsDir)) { + console.warn( + `Skipping ${owner}/${repo}@${gitRef}: ${repoDocsDir} directory does not exist.` + ); + return; + } + + integrateRepoDocs({ + owner, + repo, + docsDir, + repoDocsDir, + assetsRoot, + includeFile: () => true, + addMarkdownDoc: ({ + file, + filePath, + slugPath, + title, + navTitle, + sortRank, + hideInNav, + }) => { + const slug = path.join(slugPrefix, slugPath); + + docsCollection[slug] = { + type: "unversioned-repo-doc", + slug, + filePath: file, + owner, + repo, + gitRef, + slugPrefix, + repoFilePath: filePath, + assetsRoot, + title, + navTitle: slugPath === "" ? repo : navTitle, + sortRank, + hideInNav, + children: [], + }; + }, + }); +}; + +const fetchRepoDocs = async (source: GithubMarkdownSource) => { + switch (source.versioning) { + case "release-branches": + await fetchVersionedRepoDocs(source); + break; + case "latest-release-tag": + await fetchLatestReleaseTagRepoDocs(source); + break; + } }; for (const sourceConfig of docsConfig.githubMarkdownSources) { diff --git a/src/app/docs/LeftNav.tsx b/src/app/docs/LeftNav.tsx index 8ff800107..3c6f3c510 100644 --- a/src/app/docs/LeftNav.tsx +++ b/src/app/docs/LeftNav.tsx @@ -21,6 +21,7 @@ import { IconChartLine, IconMap, IconFileDescription, + IconPackage, IconProps, IconTag, } from "@tabler/icons-react"; @@ -37,6 +38,7 @@ const iconMap: Record> = { "chart-line": IconChartLine, map: IconMap, "file-description": IconFileDescription, + package: IconPackage, }; function NavIcon({ iconName, ...props }: { iconName: string } & IconProps) { @@ -86,6 +88,10 @@ function buildRecursiveNav( return true; } + if (child.type === "unversioned-repo-doc") { + return true; + } + // Always show latest version docs if we're not looking at a different version of the same repo. if ( !currentPageVersion && @@ -122,7 +128,12 @@ function buildRecursiveNav(