Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
[![Total Downloads](https://poser.pugx.org/myerscode/beacon/downloads)](https://packagist.org/packages/myerscode/beacon)
[![PHP Version Require](http://poser.pugx.org/myerscode/beacon/require/php)](https://packagist.org/packages/myerscode/beacon)
[![License](https://poser.pugx.org/myerscode/beacon/license)](https://github.com/myerscode/beacon/blob/main/LICENSE)
[![Tests](https://github.com/myerscode/beacon/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/myerscode/beacon/actions/workflows/tests.yml)
[![Tests](https://github.com/myerscode/beacon/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/myerscode/beacon/actions/workflows/tests.yml)
[![codecov](https://codecov.io/gh/myerscode/beacon/graph/badge.svg)](https://codecov.io/gh/myerscode/beacon)

# Beacon
Expand Down
2 changes: 1 addition & 1 deletion src/Browser.php
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ public function visitAll(array $urls, int $concurrency = 5): array

$slotCount = min($concurrency, count($urls));

/** @var array<int, Fiber> $fibers keyed by slot */
/** @var array<int, Fiber<null, null, Page, null>> $fibers keyed by slot */
$fibers = [];

/** @var array<int, int> $fiberIndex maps slot to original URL index */
Expand Down
70 changes: 35 additions & 35 deletions src/Crawler/CrawlConfig.php
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,33 @@ class CrawlConfig

private int $timeout = 30;

/**
* Calculate the request delay for a single request, applying jitter if configured.
* Returns 0 if no request delay is set.
*/
public function calculateRequestDelay(): int
{
if ($this->requestDelay <= 0) {
return 0;
}

return $this->applyJitter($this->requestDelay, $this->requestJitter);
}

/**
* Calculate the retry delay for a given attempt, applying exponential backoff and jitter.
*
* Attempt 1: retryDelay * backoffRate^0
* Attempt 2: retryDelay * backoffRate^1
* Attempt N: retryDelay * backoffRate^(N-1)
*/
public function calculateRetryDelay(int $attempt): int
{
$delay = (int) round($this->retryDelay * ($this->retryBackoff ** ($attempt - 1)));

return $this->applyJitter($delay, $this->retryJitter);
}

/**
* @param string[] $patterns URL patterns to exclude (matched with str_contains)
*/
Expand Down Expand Up @@ -80,33 +107,6 @@ public function getRequestDelay(): int
return $this->requestDelay;
}

/**
* Calculate the request delay for a single request, applying jitter if configured.
* Returns 0 if no request delay is set.
*/
public function calculateRequestDelay(): int
{
if ($this->requestDelay <= 0) {
return 0;
}

return $this->applyJitter($this->requestDelay, $this->requestJitter);
}

/**
* Calculate the retry delay for a given attempt, applying exponential backoff and jitter.
*
* Attempt 1: retryDelay * backoffRate^0
* Attempt 2: retryDelay * backoffRate^1
* Attempt N: retryDelay * backoffRate^(N-1)
*/
public function calculateRetryDelay(int $attempt): int
{
$delay = (int) round($this->retryDelay * ($this->retryBackoff ** ($attempt - 1)));

return $this->applyJitter($delay, $this->retryJitter);
}

public function getRequestJitter(): JitterStrategy
{
return $this->requestJitter;
Expand Down Expand Up @@ -227,23 +227,23 @@ public function requestWithJitter(JitterStrategy $strategy = JitterStrategy::FUL
}

/**
* Set the base delay in milliseconds between retry attempts (default: 1000ms).
* Combined with backoff rate, this determines the wait before each retry.
* Set the backoff multiplier for retry delays (default: 2.0).
* Each retry waits retryDelay * backoffRate^(attempt-1) milliseconds.
*/
public function retryDelay(int $milliseconds): self
public function retryBackoff(float $rate): self
{
$this->retryDelay = $milliseconds;
$this->retryBackoff = $rate;

return $this;
}

/**
* Set the backoff multiplier for retry delays (default: 2.0).
* Each retry waits retryDelay * backoffRate^(attempt-1) milliseconds.
* Set the base delay in milliseconds between retry attempts (default: 1000ms).
* Combined with backoff rate, this determines the wait before each retry.
*/
public function retryBackoff(float $rate): self
public function retryDelay(int $milliseconds): self
{
$this->retryBackoff = $rate;
$this->retryDelay = $milliseconds;

return $this;
}
Expand Down
2 changes: 1 addition & 1 deletion src/Crawler/JitterStrategy.php
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,6 @@
*/
enum JitterStrategy: string
{
case NONE = 'none';
case FULL = 'full';
case NONE = 'none';
}
132 changes: 66 additions & 66 deletions src/Driver/ChromeDriverInstaller.php
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@ public static function make(): static
return new static();
}

public function binaryName(): string
{
return PHP_OS_FAMILY === 'Windows' ? 'chromedriver.exe' : 'chromedriver';
}

public function clean(?string $driversDir = null): InstallationResult
{
$driversDir = $this->resolveDriversDir($driversDir);
Expand All @@ -49,72 +54,6 @@ public function clean(?string $driversDir = null): InstallationResult
return InstallationResult::nothing(sprintf('ChromeDriver not found in %s, nothing to clean.', $driversDir));
}

public function getChromeVersion(): ?string
{
$candidates = PHP_OS_FAMILY === 'Windows'
? $this->windowsChromePaths()
: $this->unixChromePaths();

foreach ($candidates as $candidate) {
$version = $this->getBinaryVersion($candidate);

if ($version !== null) {
return $version;
}
}

$finder = new ExecutableFinder();
$names = ['google-chrome', 'google-chrome-stable', 'chromium', 'chromium-browser', 'chrome'];

foreach ($names as $name) {
$found = $finder->find($name);

if ($found !== null) {
$version = $this->getBinaryVersion($found);

if ($version !== null) {
return $version;
}
}
}

return null;
}

public function install(bool $force = false, ?string $driversDir = null): InstallationResult
{
$driversDir = $this->resolveDriversDir($driversDir);
$binary = $driversDir.DIRECTORY_SEPARATOR.$this->binaryName();

if (! $force && file_exists($binary)) {
$installedVersion = $this->getBinaryVersion($binary);
$chromeVersion = $this->getChromeVersion();

if ($installedVersion !== null && $chromeVersion !== null) {
$installedMajor = (int) explode('.', $installedVersion)[0];
$chromeMajor = (int) explode('.', $chromeVersion)[0];

if ($installedMajor === $chromeMajor) {
return InstallationResult::skipped(
sprintf('ChromeDriver %s already installed and matches Chrome %s. Skipping.', $installedVersion, $chromeVersion),
);
}
}
}

return $this->download($driversDir);
}

public function update(?string $driversDir = null): InstallationResult
{
return $this->install(force: true, driversDir: $driversDir);
}

public function binaryName(): string
{
return PHP_OS_FAMILY === 'Windows' ? 'chromedriver.exe' : 'chromedriver';
}

public function download(string $driversDir): InstallationResult
{
$chromeVersion = $this->getChromeVersion();
Expand Down Expand Up @@ -193,6 +132,62 @@ public function getBinaryVersion(string $binary): ?string
return null;
}

public function getChromeVersion(): ?string
{
$candidates = PHP_OS_FAMILY === 'Windows'
? $this->windowsChromePaths()
: $this->unixChromePaths();

foreach ($candidates as $candidate) {
$version = $this->getBinaryVersion($candidate);

if ($version !== null) {
return $version;
}
}

$finder = new ExecutableFinder();
$names = ['google-chrome', 'google-chrome-stable', 'chromium', 'chromium-browser', 'chrome'];

foreach ($names as $name) {
$found = $finder->find($name);

if ($found !== null) {
$version = $this->getBinaryVersion($found);

if ($version !== null) {
return $version;
}
}
}

return null;
}

public function install(bool $force = false, ?string $driversDir = null): InstallationResult
{
$driversDir = $this->resolveDriversDir($driversDir);
$binary = $driversDir.DIRECTORY_SEPARATOR.$this->binaryName();

if (! $force && file_exists($binary)) {
$installedVersion = $this->getBinaryVersion($binary);
$chromeVersion = $this->getChromeVersion();

if ($installedVersion !== null && $chromeVersion !== null) {
$installedMajor = (int) explode('.', $installedVersion)[0];
$chromeMajor = (int) explode('.', $chromeVersion)[0];

if ($installedMajor === $chromeMajor) {
return InstallationResult::skipped(
sprintf('ChromeDriver %s already installed and matches Chrome %s. Skipping.', $installedVersion, $chromeVersion),
);
}
}
}

return $this->download($driversDir);
}

public function platform(): string
{
$os = PHP_OS_FAMILY;
Expand Down Expand Up @@ -220,6 +215,11 @@ public function resolveDriversDir(?string $dir = null): string
return $dir;
}

public function update(?string $driversDir = null): InstallationResult
{
return $this->install(force: true, driversDir: $driversDir);
}

private function extractBinary(string $zipPath, string $driversDir): void
{
if (! extension_loaded('zip')) {
Expand Down
4 changes: 3 additions & 1 deletion src/Driver/HttpClientInterface.php
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,14 @@

namespace Myerscode\Beacon\Driver;

use RuntimeException;

interface HttpClientInterface
{
/**
* Fetch the contents of a URL.
*
* @throws \RuntimeException If the request fails.
* @throws RuntimeException If the request fails.
*/
public function fetch(string $url): string;
}
Loading
Loading