diff --git a/.github/workflows/k3d-ci.yaml b/.github/workflows/k3d-ci.yaml index 30466d86a3..6e5ac7185e 100644 --- a/.github/workflows/k3d-ci.yaml +++ b/.github/workflows/k3d-ci.yaml @@ -45,6 +45,13 @@ jobs: needs: paths-filter if: needs.paths-filter.outputs.matches == 'true' steps: + - name: Initial Disk Cleanup + uses: mathio/gha-cleanup@v1 + with: + remove-browsers: true + verbose: true + + - name: Create k3d Cluster uses: AbsaOSS/k3d-action@v2 with: diff --git a/frontend/src/components/ui/badge.ts b/frontend/src/components/ui/badge.ts index adc17f2b12..e679229211 100644 --- a/frontend/src/components/ui/badge.ts +++ b/frontend/src/components/ui/badge.ts @@ -14,7 +14,8 @@ export type BadgeVariant = | "cyan" | "blue" | "high-contrast" - | "text"; + | "text" + | "text-neutral"; /** * Show numeric value in a label @@ -66,6 +67,7 @@ export class Badge extends TailwindElement { cyan: tw`bg-cyan-50 text-cyan-600 ring-cyan-600`, blue: tw`bg-blue-50 text-blue-600 ring-blue-600`, text: tw`text-blue-500 ring-blue-600`, + "text-neutral": tw`text-neutral-500 ring-neutral-600`, }[this.variant], ] : { @@ -78,6 +80,7 @@ export class Badge extends TailwindElement { cyan: tw`bg-cyan-50 text-cyan-600`, blue: tw`bg-blue-50 text-blue-600`, text: tw`text-blue-500`, + "text-neutral": tw`text-neutral-500`, }[this.variant], this.pill ? [ diff --git a/frontend/src/components/ui/config-details.ts b/frontend/src/components/ui/config-details.ts index 6d53d36688..2228e54502 100644 --- a/frontend/src/components/ui/config-details.ts +++ b/frontend/src/components/ui/config-details.ts @@ -1,3 +1,4 @@ +import { consume } from "@lit/context"; import { localized, msg, str } from "@lit/localize"; import ISO6391 from "iso-639-1"; import { html, nothing, type TemplateResult } from "lit"; @@ -6,6 +7,10 @@ import { when } from "lit/directives/when.js"; import capitalize from "lodash/fp/capitalize"; import { BtrixElement } from "@/classes/BtrixElement"; +import { + orgProxiesContext, + type OrgProxiesContext, +} from "@/context/org-proxies"; import { none, notSpecified } from "@/layouts/empty"; import { Behavior, @@ -36,6 +41,9 @@ import { getServerDefaults } from "@/utils/workflow"; @customElement("btrix-config-details") @localized() export class ConfigDetails extends BtrixElement { + @consume({ context: orgProxiesContext, subscribe: true }) + private readonly orgProxies?: OrgProxiesContext; + @property({ type: Object }) crawlConfig?: CrawlConfig; @@ -235,16 +243,16 @@ export class ConfigDetails extends BtrixElement { msg("Browser Profile"), when( crawlConfig?.profileid, - () => - html` html` + ${crawlConfig?.profileName} - `, + + `, () => crawlConfig?.profileName || html``, ), )} + ${crawlConfig?.proxyId + ? this.renderSetting( + msg("Crawler Proxy Server"), + this.orgProxies?.servers.find( + ({ id }) => id === crawlConfig.proxyId, + )?.label || capitalize(crawlConfig.proxyId), + ) + : nothing} ${this.renderSetting( msg("Browser Windows"), crawlConfig?.browserWindows ? `${crawlConfig.browserWindows}` : "", @@ -284,9 +300,6 @@ export class ConfigDetails extends BtrixElement { ISO6391.getName(seedsConfig.lang), ) : nothing} - ${crawlConfig?.proxyId - ? this.renderSetting(msg("Proxy"), capitalize(crawlConfig.proxyId)) - : nothing} `, })} ${this.renderSection({ diff --git a/frontend/src/components/ui/link.ts b/frontend/src/components/ui/link.ts index c833207030..73f1c46c45 100644 --- a/frontend/src/components/ui/link.ts +++ b/frontend/src/components/ui/link.ts @@ -1,5 +1,5 @@ import clsx from "clsx"; -import { html } from "lit"; +import { html, nothing } from "lit"; import { customElement, property } from "lit/decorators.js"; import { ifDefined } from "lit/directives/if-defined.js"; @@ -19,13 +19,16 @@ export class Link extends BtrixElement { @property({ type: String }) variant: "primary" | "neutral" = "neutral"; + @property({ type: Boolean }) + hideIcon = false; + render() { if (!this.href) return; return html` - + ${this.hideIcon + ? nothing + : html` + + `} + `; } } diff --git a/frontend/src/components/ui/select-crawler-proxy.ts b/frontend/src/components/ui/select-crawler-proxy.ts index abe19b5a76..8c5c5529ce 100644 --- a/frontend/src/components/ui/select-crawler-proxy.ts +++ b/frontend/src/components/ui/select-crawler-proxy.ts @@ -1,6 +1,6 @@ import { localized, msg } from "@lit/localize"; import type { SlSelect } from "@shoelace-style/shoelace"; -import { html } from "lit"; +import { html, type PropertyValues } from "lit"; import { customElement, property, state } from "lit/decorators.js"; import { ifDefined } from "lit/directives/if-defined.js"; @@ -40,15 +40,30 @@ export class SelectCrawlerProxy extends BtrixElement { @property({ type: String }) defaultProxyId: string | null = null; + @property({ type: String }) + profileProxyId?: string | null = null; + @property({ type: Array }) proxyServers: Proxy[] = []; @property({ type: String }) proxyId: string | null = null; + @property({ type: String }) + label?: string; + @property({ type: String }) size?: SlSelect["size"]; + @property({ type: String }) + placeholder?: string; + + @property({ type: String }) + helpText?: string; + + @property({ type: Boolean }) + disabled?: boolean; + @state() private selectedProxy?: Proxy; @@ -59,6 +74,18 @@ export class SelectCrawlerProxy extends BtrixElement { return this.selectedProxy?.id || ""; } + protected willUpdate(changedProperties: PropertyValues): void { + if (changedProperties.has("proxyId")) { + if (this.proxyId) { + this.selectedProxy = this.proxyServers.find( + ({ id }) => id === this.proxyId, + ); + } else if (changedProperties.get("proxyId")) { + this.selectedProxy = undefined; + } + } + } + protected firstUpdated() { void this.initProxies(); } @@ -75,18 +102,21 @@ export class SelectCrawlerProxy extends BtrixElement { return html` + + ${this.proxyServers.map( (server) => html` @@ -118,6 +148,7 @@ export class SelectCrawlerProxy extends BtrixElement { ` : ``} + `; } diff --git a/frontend/src/controllers/localize.ts b/frontend/src/controllers/localize.ts index 328d780a14..f70bb3aecb 100644 --- a/frontend/src/controllers/localize.ts +++ b/frontend/src/controllers/localize.ts @@ -55,7 +55,11 @@ export class LocalizeController extends SlLocalizeController { }) : seconds > 60 ? html`` diff --git a/frontend/src/features/browser-profiles/new-browser-profile-dialog.ts b/frontend/src/features/browser-profiles/new-browser-profile-dialog.ts index d5c1e1f86a..ff1bc17d08 100644 --- a/frontend/src/features/browser-profiles/new-browser-profile-dialog.ts +++ b/frontend/src/features/browser-profiles/new-browser-profile-dialog.ts @@ -116,50 +116,53 @@ export class NewBrowserProfileDialog extends BtrixElement { > + ${showProxies + ? html` +
+ + (this.proxyId = e.detail.value)} + > +
+ ${msg( + "When a proxy is selected, websites will see traffic as coming from the IP address of the proxy rather than where Browsertrix is deployed.", + )} +
+ +
+ ` + : nothing} + - ${when( - showChannels || showProxies, - () => html` - - ${msg("Crawler Settings")} - - ${showChannels - ? html`
- - (this.crawlerChannel = e.detail.value!)} - > -
` - : nothing} - ${showProxies - ? html` -
- - (this.proxyId = e.detail.value)} - > -
- ` - : nothing} - - `, - )} + ${showChannels + ? html` + ${msg("Browser Session Settings")} +
+ + (this.crawlerChannel = e.detail.value!)} + >
` + : nothing} diff --git a/frontend/src/features/browser-profiles/select-browser-profile.ts b/frontend/src/features/browser-profiles/select-browser-profile.ts index 54d602972d..9f37619ce4 100644 --- a/frontend/src/features/browser-profiles/select-browser-profile.ts +++ b/frontend/src/features/browser-profiles/select-browser-profile.ts @@ -1,18 +1,41 @@ import { localized, msg } from "@lit/localize"; -import { type SlSelect } from "@shoelace-style/shoelace"; -import { html, nothing, type PropertyValues } from "lit"; -import { customElement, property, state } from "lit/decorators.js"; +import { Task } from "@lit/task"; +import type { + SlChangeEvent, + SlDrawer, + SlSelect, +} from "@shoelace-style/shoelace"; +import clsx from "clsx"; +import { html, nothing } from "lit"; +import { customElement, property, query, state } from "lit/decorators.js"; import { ifDefined } from "lit/directives/if-defined.js"; -import orderBy from "lodash/fp/orderBy"; +import { when } from "lit/directives/when.js"; +import queryString from "query-string"; + +import { originsWithRemainder } from "./templates/origins-with-remainder"; import { BtrixElement } from "@/classes/BtrixElement"; -import type { Profile } from "@/pages/org/types"; -import type { APIPaginatedList } from "@/types/api"; +import { none } from "@/layouts/empty"; +import { pageHeading } from "@/layouts/page"; +import { CrawlerChannelImage, type Profile } from "@/pages/org/types"; +import { OrgTab } from "@/routes"; +import type { + APIPaginatedList, + APIPaginationQuery, + APISortQuery, +} from "@/types/api"; +import { SortDirection } from "@/types/utils"; +import { isNotEqual } from "@/utils/is-not-equal"; +import { AppStateService } from "@/utils/state"; +import { tw } from "@/utils/tailwind"; type SelectBrowserProfileChangeDetail = { value: Profile | undefined; }; +// TODO Paginate results +const INITIAL_PAGE_SIZE = 1000; + export type SelectBrowserProfileChangeEvent = CustomEvent; @@ -37,130 +60,311 @@ export class SelectBrowserProfile extends BtrixElement { @property({ type: String }) profileId?: string; - @state() - private selectedProfile?: Profile; + @property({ type: String }) + profileName?: string; + + /** + * List of origins to match to prioritize profile options + */ + @property({ type: Array, hasChanged: isNotEqual }) + suggestOrigins?: string[]; @state() - private browserProfiles?: Profile[]; + selectedProfile?: Profile; - willUpdate(changedProperties: PropertyValues) { - if (changedProperties.has("profileId")) { - void this.updateSelectedProfile(); - } + @query("sl-select") + private readonly select?: SlSelect | null; + + @query("sl-drawer") + private readonly drawer?: SlDrawer | null; + + public get value() { + return this.select?.value as string; } - firstUpdated() { - void this.updateSelectedProfile(); + private readonly profilesTask = new Task(this, { + task: async (_args, { signal }) => { + return this.getProfiles( + { + sortBy: "name", + sortDirection: SortDirection.Ascending, + pageSize: INITIAL_PAGE_SIZE, + }, + signal, + ); + }, + args: () => [] as const, + }); + + private readonly selectedProfileTask = new Task(this, { + task: async ([profileId, profiles], { signal }) => { + if (!profileId || !profiles || signal.aborted) return; + + this.selectedProfile = this.findProfileById(profileId); + }, + args: () => [this.profileId, this.profilesTask.value] as const, + }); + + private findProfileById(profileId?: string) { + if (!profileId) return; + return this.profilesTask.value?.items.find(({ id }) => id === profileId); } render() { + const selectedProfile = this.selectedProfile; + const browserProfiles = this.profilesTask.value; + const loading = !browserProfiles && !this.profileName; + return html` { - // Refetch to keep list up to date - void this.fetchBrowserProfiles(); - }} @sl-hide=${this.stopProp} @sl-after-hide=${this.stopProp} > - ${this.browserProfiles - ? html` - ${msg("No custom profile")} - - ` - : html` `} - ${this.browserProfiles?.map( - (profile) => html` - - ${profile.name} -
-
- -
- `, - )} - ${this.browserProfiles && !this.browserProfiles.length + ${loading ? html`` : nothing} + ${this.renderProfileOptions()} + ${browserProfiles && !browserProfiles.total ? this.renderNoProfiles() : ""}
- ${this.selectedProfile + ${selectedProfile ? html` + - ${msg("Last updated")} - + ${msg("Last saved")} + ${this.localize.relativeDate( + selectedProfile.modified || selectedProfile.created, + { capitalize: true }, + )} - ${this.selectedProfile.proxyId - ? html` - ${msg("Using proxy: ")} - ${this.selectedProfile.proxyId} - ` - : ``} - - ${msg("Check Profile")} - - ` - : this.browserProfiles + : browserProfiles ? html` - - ${msg("View Profiles")} - - + ${msg("View Browser Profiles")} + ` : nothing}
- ${this.browserProfiles?.length ? this.renderSelectedProfileInfo() : ""} + ${browserProfiles || selectedProfile + ? this.renderSelectedProfileInfo() + : ""} `; } - private renderSelectedProfileInfo() { - if (!this.selectedProfile?.description) return; + private renderProfileOptions() { + const browserProfiles = this.profilesTask.value; - return html`
- -
- ${msg("Description")} -
- -
${this.selectedProfile.description}
+ ${this.profileName} + `; + } + + return; + } + + const option = (profile: Profile, i: number) => html` + +
${this.renderOverview(profile)}
+ + - -
`; + ${profile.name} +
+ ${originsWithRemainder(profile.origins, { + disablePopover: true, + })} +
+ + + `; + + const profiles = browserProfiles.items; + const priorityOrigins = this.suggestOrigins; + const suggestions: Profile[] = []; + let rest: Profile[] = []; + + if (priorityOrigins?.length) { + profiles.forEach((profile) => { + const { origins } = profile; + if ( + origins.some((origin) => + priorityOrigins.includes( + new URL(origin).hostname.replace(/^www\./, ""), + ), + ) + ) { + suggestions.push(profile); + } else { + rest.push(profile); + } + }); + } else { + rest = profiles; + } + + return html` + ${msg("No custom profile")} + ${suggestions.length + ? html` + + ${msg("Suggested Profiles")} + ${suggestions.map(option)} + ` + : nothing} + ${rest.length + ? html` + + ${suggestions.length + ? msg("Other Saved Profiles") + : msg("Saved Profiles")} + ${rest.map(option)} + ` + : nothing} + `; + } + + private renderSelectedProfileInfo() { + const profileContent = (profile: Profile) => { + return html`${pageHeading({ content: msg("Overview"), level: 3 })} +
${this.renderOverview(profile)}
+ + + + ${pageHeading({ content: msg("Saved Sites"), level: 3 })} +
+ ${profile.origins.length + ? html`
    + ${profile.origins.map( + (origin) => html` +
  • + +
    • + `, + )} +
` + : none} +
+ +
+ + ${msg("View More")} + +
`; + }; + + return html` { + // Hide any other open panels + AppStateService.updateUserGuideOpen(false); + }} + > + + + ${this.selectedProfile?.name} + + + ${when(this.selectedProfile, profileContent)} + `; } + private readonly renderOverview = (profile: Profile) => { + const modifiedByAnyDate = [ + profile.modifiedCrawlDate, + profile.modified, + profile.created, + ].reduce((a, b) => (b && a && b > a ? b : a), profile.created); + + return html` + + ${profile.description + ? html` + +
${profile.description}
+ ` + : none} + + + ${profile.tags.length + ? html`
+ ${profile.tags.map((tag) => html`${tag}`)} +
` + : none} + + + + + ${when( + profile.proxyId, + (proxyId) => html` + + + + `, + )} + + ${this.localize.relativeDate(modifiedByAnyDate || profile.created, { + capitalize: true, + })} + + `; + }; + private renderNoProfiles() { return html`
@@ -194,10 +398,9 @@ export class SelectBrowserProfile extends BtrixElement { `; } - private async onChange(e: Event) { - this.selectedProfile = this.browserProfiles?.find( - ({ id }) => id === (e.target as SlSelect | null)?.value, - ); + private async onChange(e: SlChangeEvent) { + const profileId = (e.target as SlSelect | null)?.value as string; + this.selectedProfile = this.findProfileById(profileId); await this.updateComplete; @@ -210,43 +413,30 @@ export class SelectBrowserProfile extends BtrixElement { ); } - private async updateSelectedProfile() { - await this.fetchBrowserProfiles(); - await this.updateComplete; - - if (this.profileId && !this.selectedProfile) { - this.selectedProfile = this.browserProfiles?.find( - ({ id }) => id === this.profileId, - ); - } - } - - /** - * Fetch browser profiles and update internal state - */ - private async fetchBrowserProfiles(): Promise { - try { - const data = await this.getProfiles(); - - this.browserProfiles = orderBy(["name", "modified"])(["asc", "desc"])( - data, - ) as Profile[]; - } catch (e) { - this.notify.toast({ - message: msg("Sorry, couldn't retrieve browser profiles at this time."), - variant: "danger", - icon: "exclamation-octagon", - id: "browser-profile-status", - }); - } - } + private async getProfiles( + params: { + userid?: string; + tags?: string[]; + tagMatch?: string; + } & APIPaginationQuery & + APISortQuery, + signal: AbortSignal, + ) { + const query = queryString.stringify( + { + ...params, + }, + { + arrayFormat: "none", // For tags + }, + ); - private async getProfiles() { const data = await this.api.fetch>( - `/orgs/${this.orgId}/profiles`, + `/orgs/${this.orgId}/profiles?${query}`, + { signal }, ); - return data.items; + return data; } /** diff --git a/frontend/src/features/browser-profiles/start-browser-dialog.ts b/frontend/src/features/browser-profiles/start-browser-dialog.ts index 877f15acd3..a9b9fa2a11 100644 --- a/frontend/src/features/browser-profiles/start-browser-dialog.ts +++ b/frontend/src/features/browser-profiles/start-browser-dialog.ts @@ -230,41 +230,33 @@ export class StartBrowserDialog extends BtrixElement { )} - ${when( - this.open && (showChannels || showProxies), - () => html` - - ${msg("Crawler Settings")} - - ${showChannels - ? html`
- - -
` - : nothing} - ${showProxies - ? html`
- - -
` - : nothing} - - `, - )} + ${showProxies + ? html`
+ + +
` + : nothing} + ${this.open && showChannels + ? html` + ${msg("Browser Session Settings")} +
+ + +
+ ` + : nothing}
void this.dialog?.hide()} @@ -318,7 +310,7 @@ export class StartBrowserDialog extends BtrixElement { ${msg("Suggestions from Related Workflows")} - ${seeds.map(option)} + ${seeds.slice(0, 10).map(option)} ` : nothing, )} diff --git a/frontend/src/features/browser-profiles/templates/badges.ts b/frontend/src/features/browser-profiles/templates/badges.ts index 303b2810dd..5fdea74f6f 100644 --- a/frontend/src/features/browser-profiles/templates/badges.ts +++ b/frontend/src/features/browser-profiles/templates/badges.ts @@ -1,12 +1,15 @@ import { msg } from "@lit/localize"; import { html, nothing } from "lit"; import { when } from "lit/directives/when.js"; -import capitalize from "lodash/fp/capitalize"; -import { CrawlerChannelImage, type Profile } from "@/types/crawler"; +import { type Profile } from "@/types/crawler"; export const usageBadge = (inUse: boolean) => - html` + html` - html` - - - ${capitalize(channel)} - - `, + (channelImage) => html` + + `, )} ${when( profile.proxyId, - (proxy) => - html` - - - ${proxy} - - `, + (proxyId) => html` + + `, )}
`; }; diff --git a/frontend/src/features/browser-profiles/templates/origins-with-remainder.ts b/frontend/src/features/browser-profiles/templates/origins-with-remainder.ts new file mode 100644 index 0000000000..17080a26d9 --- /dev/null +++ b/frontend/src/features/browser-profiles/templates/origins-with-remainder.ts @@ -0,0 +1,39 @@ +import { html, nothing } from "lit"; + +import type { Profile } from "@/types/crawler"; +import localize from "@/utils/localize"; + +/** + * Displays primary origin with remainder in a popover badge + */ +export function originsWithRemainder( + origins: Profile["origins"], + { disablePopover } = { disablePopover: false }, +) { + const startingUrl = origins[0]; + const otherOrigins = origins.slice(1); + + return html`
+ + ${otherOrigins.length + ? html` + + +${localize.number(otherOrigins.length)} +
    + ${otherOrigins.map((url) => html`
  • ${url}
    • `)} +
+ + ` + : nothing} +
`; +} diff --git a/frontend/src/features/crawl-workflows/workflow-editor.ts b/frontend/src/features/crawl-workflows/workflow-editor.ts index 68942af1eb..28a0df4d48 100644 --- a/frontend/src/features/crawl-workflows/workflow-editor.ts +++ b/frontend/src/features/crawl-workflows/workflow-editor.ts @@ -1,5 +1,6 @@ import { consume } from "@lit/context"; import { localized, msg, str } from "@lit/localize"; +import { Task } from "@lit/task"; import type { SlBlurEvent, SlChangeEvent, @@ -33,6 +34,7 @@ import { state, } from "lit/decorators.js"; import { choose } from "lit/directives/choose.js"; +import { guard } from "lit/directives/guard.js"; import { ifDefined } from "lit/directives/if-defined.js"; import { map } from "lit/directives/map.js"; import { when } from "lit/directives/when.js"; @@ -95,6 +97,7 @@ import { Behavior, CrawlerChannelImage, ScopeType, + type Profile, type Seed, type WorkflowParams, } from "@/types/crawler"; @@ -419,6 +422,15 @@ export class WorkflowEditor extends BtrixElement { // https://github.com/webrecorder/browsertrix-crawler/blob/v1.5.8/package.json#L23 private readonly cssParser = createParser(); + private readonly profileTask = new Task(this, { + task: async ([formState], { signal }) => { + if (!formState.browserProfile) return; + + return this.getProfile(formState.browserProfile.id, signal); + }, + args: () => [this.formState] as const, + }); + connectedCallback(): void { this.initializeEditor(); super.connectedCallback(); @@ -1982,15 +1994,49 @@ https://archiveweb.page/images/${"logo.svg"}`} if (!this.formState.lang) throw new Error("missing formstate.lang"); const proxies = this.proxies; + const profileProxyId = + this.formState.browserProfile?.proxyId || + (this.formState.browserProfile?.id && this.formState.proxyId); + + const priorityOrigins = () => { + if (!this.formState.urlList && !this.formState.primarySeedUrl) { + return []; + } + + const crawlUrls = urlListToArray(this.formState.urlList); + + if (this.formState.primarySeedUrl) { + crawlUrls.unshift(this.formState.primarySeedUrl); + } + + return crawlUrls + .map((url) => { + try { + return new URL(url).hostname.replace(/^www\./, ""); + } catch { + return ""; + } + }) + .filter((url) => url); + }; return html` ${inputCol(html` + .profileName=${this.formState.browserProfile?.name} + .suggestOrigins=${guard( + [this.formState.primarySeedUrl, this.formState.urlList], + priorityOrigins, + )} + @on-change=${(e: SelectBrowserProfileChangeEvent) => { + const profile = e.detail.value; + this.updateFormState({ - browserProfile: e.detail.value ?? null, - })} + browserProfile: profile ?? null, + proxyId: profile?.proxyId ?? null, + }); + }} > `)} ${this.renderHelpTextCol(infoTextFor["browserProfile"])} @@ -2002,12 +2048,24 @@ https://archiveweb.page/images/${"logo.svg"}`} proxies.default_proxy_id ?? undefined, )} .proxyServers=${proxies.servers} - .proxyId="${this.formState.proxyId || ""}" + .proxyId=${profileProxyId || this.formState.proxyId || ""} + .profileProxyId=${profileProxyId} @btrix-change=${(e: SelectCrawlerProxyChangeEvent) => this.updateFormState({ proxyId: e.detail.value, })} - > + > + ${when( + profileProxyId, + () => html` + ${msg("Set by profile")} + `, + )} + `), this.renderHelpTextCol(infoTextFor["proxyId"]), ] @@ -3253,7 +3311,7 @@ https://archiveweb.page/images/${"logo.svg"}`} }, crawlerChannel: this.formState.crawlerChannel || CrawlerChannelImage.Default, - proxyId: this.formState.proxyId, + proxyId: this.formState.browserProfile?.proxyId || this.formState.proxyId, }; return config; @@ -3410,4 +3468,13 @@ https://archiveweb.page/images/${"logo.svg"}`} console.debug(e); } } + + private async getProfile(profileId: string, signal: AbortSignal) { + const data = await this.api.fetch( + `/orgs/${this.orgId}/profiles/${profileId}`, + { signal }, + ); + + return data; + } } diff --git a/frontend/src/features/crawls/crawler-channel-badge.ts b/frontend/src/features/crawls/crawler-channel-badge.ts new file mode 100644 index 0000000000..adce184bdc --- /dev/null +++ b/frontend/src/features/crawls/crawler-channel-badge.ts @@ -0,0 +1,46 @@ +import { consume } from "@lit/context"; +import { localized } from "@lit/localize"; +import { html } from "lit"; +import { customElement, property } from "lit/decorators.js"; +import { ifDefined } from "lit/directives/if-defined.js"; + +import { TailwindElement } from "@/classes/TailwindElement"; +import { + orgCrawlerChannelsContext, + type OrgCrawlerChannelsContext, +} from "@/context/org-crawler-channels"; +import { CrawlerChannelImage } from "@/types/crawler"; + +@customElement("btrix-crawler-channel-badge") +@localized() +export class CrawlerChannelBadge extends TailwindElement { + @consume({ context: orgCrawlerChannelsContext, subscribe: true }) + private readonly crawlerChannels?: OrgCrawlerChannelsContext; + + @property({ type: String }) + channelId?: CrawlerChannelImage | AnyString; + + render() { + if (!this.channelId || !this.crawlerChannels) return; + + const crawlerChannel = this.crawlerChannels.find( + ({ id }) => id === this.channelId, + ); + + return html` + + + ${this.channelId} + + `; + } +} diff --git a/frontend/src/features/crawls/index.ts b/frontend/src/features/crawls/index.ts index 5254170741..c19583ee20 100644 --- a/frontend/src/features/crawls/index.ts +++ b/frontend/src/features/crawls/index.ts @@ -1,2 +1,4 @@ import("./crawl-list"); import("./crawl-state-filter"); +import("./crawler-channel-badge"); +import("./proxy-badge"); diff --git a/frontend/src/features/crawls/proxy-badge.ts b/frontend/src/features/crawls/proxy-badge.ts new file mode 100644 index 0000000000..dabf5082a8 --- /dev/null +++ b/frontend/src/features/crawls/proxy-badge.ts @@ -0,0 +1,38 @@ +import { consume } from "@lit/context"; +import { localized } from "@lit/localize"; +import { html } from "lit"; +import { customElement, property } from "lit/decorators.js"; +import { ifDefined } from "lit/directives/if-defined.js"; + +import { TailwindElement } from "@/classes/TailwindElement"; +import { + orgProxiesContext, + type OrgProxiesContext, +} from "@/context/org-proxies"; + +@customElement("btrix-proxy-badge") +@localized() +export class ProxyBadge extends TailwindElement { + @consume({ context: orgProxiesContext, subscribe: true }) + private readonly orgProxies?: OrgProxiesContext; + + @property({ type: String }) + proxyId?: string; + + render() { + if (!this.proxyId || !this.orgProxies) return; + + const proxy = this.orgProxies.servers.find(({ id }) => id === this.proxyId); + + return html` + + + ${proxy?.label || this.proxyId} + + `; + } +} diff --git a/frontend/src/pages/org/browser-profiles-list.ts b/frontend/src/pages/org/browser-profiles-list.ts index 394b7a2cbf..4ace718c76 100644 --- a/frontend/src/pages/org/browser-profiles-list.ts +++ b/frontend/src/pages/org/browser-profiles-list.ts @@ -1,6 +1,6 @@ import { localized, msg } from "@lit/localize"; import { Task } from "@lit/task"; -import { html, nothing, type PropertyValues } from "lit"; +import { html, type PropertyValues } from "lit"; import { customElement, state } from "lit/decorators.js"; import { when } from "lit/directives/when.js"; import queryString from "query-string"; @@ -18,6 +18,7 @@ import { parsePage, type PageChangeEvent } from "@/components/ui/pagination"; import type { BtrixChangeTagFilterEvent } from "@/components/ui/tag-filter/types"; import { ClipboardController } from "@/controllers/clipboard"; import { SearchParamsValue } from "@/controllers/searchParamsValue"; +import { originsWithRemainder } from "@/features/browser-profiles/templates/origins-with-remainder"; import { emptyMessage } from "@/layouts/emptyMessage"; import { page } from "@/layouts/page"; import { OrgTab } from "@/routes"; @@ -71,7 +72,7 @@ const columnsCss = [ "min-content", // Status "[clickable-start] minmax(min-content, 1fr)", // Name "30ch", // Tags - "minmax(max-content, 1fr)", // Origins + "40ch", // Origins "minmax(min-content, 20ch)", // Last modified "[clickable-end] min-content", // Actions ].join(" "); @@ -496,7 +497,7 @@ export class BrowserProfilesList extends BtrixElement { ${msg("Name")} ${msg("Tags")} - ${msg("Configured Sites")} + ${msg("Saved Sites")} ${msg("Last Modified")} @@ -520,8 +521,6 @@ export class BrowserProfilesList extends BtrixElement { (a, b) => (b && a && b > a ? b : a), data.created, ) || data.created; - const startingUrl = data.origins[0]; - const otherOrigins = data.origins.slice(1); return html` - - - ${otherOrigins.length - ? html` - +${this.localize.number(otherOrigins.length)} -
    - ${otherOrigins.map((url) => html`
  • ${url}
    • `)} -
- ` - : nothing} + + ${originsWithRemainder(data.origins)} - ${this.localize.relativeDate(modifiedByAnyDate)} + ${this.localize.relativeDate(modifiedByAnyDate, { capitalize: true })} ${this.renderActions(data)} diff --git a/frontend/src/pages/org/browser-profiles/profile.ts b/frontend/src/pages/org/browser-profiles/profile.ts index dc33472519..434e2e0c12 100644 --- a/frontend/src/pages/org/browser-profiles/profile.ts +++ b/frontend/src/pages/org/browser-profiles/profile.ts @@ -309,7 +309,7 @@ export class BrowserProfilesProfilePage extends BtrixElement { const archivingDisabled = isArchivingDisabled(this.org); return panel({ - heading: msg("Configured Sites"), + heading: msg("Saved Sites"), actions: this.appState.isCrawler ? html` @@ -467,7 +468,9 @@ export class BrowserProfilesProfilePage extends BtrixElement { ${this.renderDetail((profile) => - this.localize.relativeDate(modifiedByAnyDate || profile.created), + this.localize.relativeDate(modifiedByAnyDate || profile.created, { + capitalize: true, + }), )} diff --git a/frontend/src/pages/org/index.ts b/frontend/src/pages/org/index.ts index 261855e125..b3530c830f 100644 --- a/frontend/src/pages/org/index.ts +++ b/frontend/src/pages/org/index.ts @@ -493,7 +493,7 @@ export class Org extends BtrixElement { .proxyServers=${proxies.servers} .crawlerChannels=${crawlerChannels} defaultProxyId=${ifDefined( - org.crawlingDefaults?.profileid || + org.crawlingDefaults?.proxyId || proxies.default_proxy_id || undefined, )} diff --git a/frontend/src/pages/org/settings/components/crawling-defaults.ts b/frontend/src/pages/org/settings/components/crawling-defaults.ts index 38284065d1..14ce869f75 100644 --- a/frontend/src/pages/org/settings/components/crawling-defaults.ts +++ b/frontend/src/pages/org/settings/components/crawling-defaults.ts @@ -20,6 +20,7 @@ import { orgProxiesContext, type OrgProxiesContext, } from "@/context/org-proxies"; +import type { SelectBrowserProfile } from "@/features/browser-profiles/select-browser-profile"; import type { CustomBehaviorsTable } from "@/features/crawl-workflows/custom-behaviors-table"; import type { QueueExclusionTable } from "@/features/crawl-workflows/queue-exclusion-table"; import { columns, type Cols } from "@/layouts/columns"; @@ -79,6 +80,9 @@ export class OrgSettingsCrawlWorkflows extends BtrixElement { @query("btrix-language-select") languageSelect?: LanguageSelect | null; + @query("btrix-select-browser-profile") + browserProfileSelect?: SelectBrowserProfile | null; + @query("btrix-select-crawler-proxy") proxySelect?: SelectCrawlerProxy | null; @@ -362,7 +366,7 @@ export class OrgSettingsCrawlWorkflows extends BtrixElement { behaviorTimeout: parseNumber(values.behaviorTimeoutSeconds), pageExtraDelay: parseNumber(values.pageExtraDelaySeconds), blockAds: values.blockAds === "on", - profileid: values.profileid, + profileid: this.browserProfileSelect?.value || undefined, crawlerChannel: values.crawlerChannel, proxyId: this.proxySelect?.value || undefined, userAgent: values.userAgent, diff --git a/frontend/src/strings/crawl-workflows/infoText.ts b/frontend/src/strings/crawl-workflows/infoText.ts index 042aa12c84..5a04269bb6 100644 --- a/frontend/src/strings/crawl-workflows/infoText.ts +++ b/frontend/src/strings/crawl-workflows/infoText.ts @@ -38,7 +38,7 @@ export const infoTextFor = { msg(`Choose a custom profile to make use of saved cookies and logged-in accounts. Note that websites may log profiles out after a period of time.`), crawlerChannel: msg( - `Choose a Browsertrix Crawler Release Channel. If available, other versions may provide new/experimental crawling features.`, + `Choose a Browsertrix Crawler release channel. If available, other versions may provide new or experimental crawling features.`, ), blockAds: msg( html`Blocks advertising content from being loaded. Uses diff --git a/frontend/src/theme.stylesheet.css b/frontend/src/theme.stylesheet.css index 59842237dd..191cd90e5a 100644 --- a/frontend/src/theme.stylesheet.css +++ b/frontend/src/theme.stylesheet.css @@ -240,7 +240,7 @@ sl-option:not([aria-selected="true"]):not(:disabled), sl-menu-item:not([disabled]), btrix-menu-item-link { - @apply part-[base]:text-neutral-700 part-[base]:hover:bg-cyan-50/50 part-[base]:hover:text-cyan-700 part-[base]:focus-visible:bg-cyan-50/50; + @apply part-[base]:bg-white part-[base]:text-neutral-700 part-[base]:hover:bg-cyan-50/50 part-[base]:hover:text-cyan-700 part-[base]:focus:text-cyan-700 part-[base]:focus-visible:bg-cyan-50/50; } sl-option[aria-selected="true"] { @@ -411,12 +411,14 @@ .font-monostyle { @apply font-mono; font-variation-settings: var(--font-monostyle-variation); + font-size: 95%; } /* Actually monospaced font */ .font-monospace { @apply font-mono; font-variation-settings: var(--font-monospace-variation); + font-size: 95%; } .truncate { @@ -528,6 +530,11 @@ top: auto; } +html { + /* Fixes sl-input components resizing when sl-scroll-lock is removed */ + scrollbar-gutter: stable; +} + /* Ensure buttons in shadow dom inherit hover color */ [class^="hover\:text-"]::part(base):hover, [class*=" hover\:text-"]::part(base):hover { diff --git a/frontend/src/types/crawler.ts b/frontend/src/types/crawler.ts index b68c0535d1..1b7eedfad3 100644 --- a/frontend/src/types/crawler.ts +++ b/frontend/src/types/crawler.ts @@ -66,6 +66,7 @@ export type WorkflowParams = { schedule: string; browserWindows: number; profileid: string | null; + profileName?: string | null; config: SeedConfig; tags: string[]; crawlTimeout: number | null; diff --git a/frontend/src/utils/workflow.ts b/frontend/src/utils/workflow.ts index 14621dbbef..40f3449716 100644 --- a/frontend/src/utils/workflow.ts +++ b/frontend/src/utils/workflow.ts @@ -383,7 +383,10 @@ export function getInitialFormState(params: { jobName: params.initialWorkflow.name || defaultFormState.jobName, description: params.initialWorkflow.description, browserProfile: params.initialWorkflow.profileid - ? ({ id: params.initialWorkflow.profileid } as Profile) + ? ({ + id: params.initialWorkflow.profileid, + name: params.initialWorkflow.profileName, + } as Profile) : defaultFormState.browserProfile, scopeType: primarySeedConfig.scopeType as FormState["scopeType"], exclusions: seedsConfig.exclude?.length === 0 ? [""] : seedsConfig.exclude,