Skip to content

Commit d131a13

Browse files
author
Michael Babker
committed
Broken Fetch Data down into chunks (Ref #27)
1 parent cbd5fcc commit d131a13

File tree

10 files changed

+294
-50
lines changed

10 files changed

+294
-50
lines changed

‎administrator/components/com_patchtester/PatchTester/Controller/DisplayController.php‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,15 @@ public function execute()
9595
// Sanity check - Ensure our classes exist
9696
if (!class_exists($viewClass))
9797
{
98-
throw new \RuntimeException(sprintf('The view class for the %1%s view in the %2%s was not found.', $view, $format), 500);
98+
// Try to use a default view
99+
$viewClass = '\\PatchTester\\View\\Default' . ucfirst($format) . 'View';
100+
101+
if (!class_exists($viewClass))
102+
{
103+
throw new \RuntimeException(
104+
sprintf('The view class for the %1$s view in the %2$s was not found.', $view, $format), 500
105+
);
106+
}
99107
}
100108

101109
if (!class_exists($modelClass))

‎administrator/components/com_patchtester/PatchTester/Controller/FetchController.php‎

Lines changed: 32 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,26 +26,50 @@ class FetchController extends DisplayController
2626
*/
2727
public function execute()
2828
{
29+
// We don't want this request to be cached.
30+
header('Pragma: no-cache');
31+
header('Cache-Control: no-cache');
32+
header('Expires: -1');
33+
2934
try
3035
{
36+
// Fetch our page from the session
37+
$page = \JFactory::getSession()->get('com_patchtester_fetcher_page', 1);
38+
3139
// TODO - Decouple the model and context?
3240
$model = new PullsModel('com_patchtester.fetch', null, \JFactory::getDbo());
3341

3442
// Initialize the state for the model
3543
$model->setState($this->initializeState($model));
3644

37-
$model->requestFromGithub();
38-
39-
$msg = \JText::_('COM_PATCHTESTER_FETCH_SUCCESSFUL');
40-
$type = 'message';
45+
$status = $model->requestFromGithub($page);
4146
}
4247
catch (\Exception $e)
4348
{
44-
$msg = $e->getMessage();
45-
$type = 'error';
49+
$response = new \JResponseJson($e);
50+
51+
echo json_encode($response);
52+
53+
$this->getApplication()->close(1);
54+
}
55+
56+
// Update the UI and session now
57+
if (isset($status['page']))
58+
{
59+
\JFactory::getSession()->set('com_patchtester_fetcher_page', $status['page']);
60+
$message = \JText::sprintf('COM_PATCHTESTER_FETCH_PAGE_NUMBER', $status['page']);
61+
unset($status['page']);
62+
}
63+
else
64+
{
65+
$status['header'] = \JText::_('COM_PATCHTESTER_FETCH_SUCCESSFUL', true);
66+
$message = \JText::_('COM_PATCHTESTER_FETCH_COMPLETE_CLOSE_WINDOW', true);
4667
}
4768

48-
$this->getApplication()->enqueueMessage($msg, $type);
49-
$this->getApplication()->redirect(\JRoute::_('index.php?option=com_patchtester', false));
69+
$response = new \JResponseJson($status, $message, false, true);
70+
71+
echo json_encode($response);
72+
73+
$this->getApplication()->close();
5074
}
5175
}

‎administrator/components/com_patchtester/PatchTester/Controller/StartfetchController.php‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
<?php
2+
/**
3+
* Patch testing component for the Joomla! CMS
4+
*
5+
* @copyright Copyright (C) 2011 - 2012 Ian MacLennan, Copyright (C) 2013 - 2015 Open Source Matters, Inc. All rights reserved.
6+
* @license GNU General Public License version 2 or later
7+
*/
8+
9+
namespace PatchTester\Controller;
10+
11+
use PatchTester\Helper;
12+
use PatchTester\Model\PullsModel;
13+
14+
/**
15+
* Controller class to start fetching remote data
16+
*
17+
* @since 2.0
18+
*/
19+
class StartfetchController extends DisplayController
20+
{
21+
/**
22+
* Execute the controller.
23+
*
24+
* @return void Redirects the application
25+
*
26+
* @since 2.0
27+
*/
28+
public function execute()
29+
{
30+
// We don't want this request to be cached.
31+
header('Pragma: no-cache');
32+
header('Cache-Control: no-cache');
33+
header('Expires: -1');
34+
35+
// Check for a valid token. If invalid, send a 403 with the error message.
36+
if (!\JSession::checkToken('request'))
37+
{
38+
$response = new \JResponseJson(new \Exception(\JText::_('JINVALID_TOKEN'), 403));
39+
40+
echo json_encode($response);
41+
42+
$this->getApplication()->close(1);
43+
}
44+
45+
// Make sure we can fetch the data from GitHub - throw an error on < 10 available requests
46+
$github = Helper::initializeGithub();
47+
$rate = $github->authorization->getRateLimit();
48+
49+
// If over the API limit, we can't build this list
50+
if ($rate->resources->core->remaining < 10)
51+
{
52+
$response = new \JResponseJson(
53+
new \Exception(
54+
\JText::sprintf('COM_PATCHTESTER_API_LIMIT_LIST', \JFactory::getDate($rate->resources->core->reset)),
55+
429
56+
)
57+
);
58+
59+
echo json_encode($response);
60+
61+
$this->getApplication()->close(1);
62+
}
63+
64+
// TODO - Decouple the model and context?
65+
$model = new PullsModel('com_patchtester.fetch', null, \JFactory::getDbo());
66+
67+
// Initialize the state for the model
68+
$model->setState($this->initializeState($model));
69+
70+
try
71+
{
72+
// Sanity check, ensure there aren't any applied patches
73+
if (count($model->getAppliedPatches()) >= 1)
74+
{
75+
$response = new \JResponseJson(new \Exception(\JText::_('COM_PATCHTESTER_ERROR_APPLIED_PATCHES'), 500));
76+
77+
echo json_encode($response);
78+
79+
$this->getApplication()->close(1);
80+
}
81+
}
82+
catch (\Exception $e)
83+
{
84+
$response = new \JResponseJson($e);
85+
86+
echo json_encode($response);
87+
88+
$this->getApplication()->close(1);
89+
}
90+
91+
// We're able to successfully pull data, prepare our environment
92+
\JFactory::getSession()->set('com_patchtester_fetcher_page', 1);
93+
94+
$response = new \JResponseJson(
95+
array('complete' => false, 'header' => \JText::_('COM_PATCHTESTER_FETCH_PROCESSING', true)),
96+
\JText::sprintf('COM_PATCHTESTER_FETCH_PAGE_NUMBER', 1),
97+
false,
98+
true
99+
);
100+
101+
echo json_encode($response);
102+
103+
$this->getApplication()->close();
104+
}
105+
}

‎administrator/components/com_patchtester/PatchTester/Model/FetchModel.php‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
<?php
2+
/**
3+
* Patch testing component for the Joomla! CMS
4+
*
5+
* @copyright Copyright (C) 2011 - 2012 Ian MacLennan, Copyright (C) 2013 - 2015 Open Source Matters, Inc. All rights reserved.
6+
* @license GNU General Public License version 2 or later
7+
*/
8+
9+
namespace PatchTester\Model;
10+
11+
/**
12+
* Model class for the fetch modal view
13+
*
14+
* @since 2.0
15+
*/
16+
class FetchModel extends PullsModel
17+
{
18+
}

‎administrator/components/com_patchtester/PatchTester/Model/PullsModel.php‎

Lines changed: 21 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -299,63 +299,42 @@ public function getTotal()
299299
/**
300300
* Method to request new data from GitHub
301301
*
302-
* @return void
302+
* @param integer $page The page of the request
303+
*
304+
* @return array
303305
*
304306
* @since 2.0
305307
* @throws \RuntimeException
306308
*/
307-
public function requestFromGithub()
309+
public function requestFromGithub($page)
308310
{
309311
// Get the Github object
310312
$github = Helper::initializeGithub();
311-
$rate = $github->authorization->getRateLimit();
312313

313-
// If over the API limit, we can't build this list
314-
if ($rate->resources->core->remaining == 0)
314+
// If on page 1, dump the old data
315+
if ($page === 1)
315316
{
316-
throw new \RuntimeException(
317-
\JText::sprintf('COM_PATCHTESTER_API_LIMIT_LIST', \JFactory::getDate($rate->resources->core->reset))
318-
);
317+
$this->getDb()->truncateTable('#__patchtester_pulls');
319318
}
320319

321-
// Sanity check, ensure there aren't any applied patches
322-
if (count($this->getAppliedPatches()) >= 1)
320+
try
323321
{
324-
throw new \RuntimeException(\JText::_('COM_PATCHTESTER_ERROR_APPLIED_PATCHES'));
322+
// TODO - Option to configure the batch size
323+
$pulls = $github->pulls->getList(
324+
$this->getState()->get('github_user'), $this->getState()->get('github_repo'), 'open', $page, 100
325+
);
325326
}
326-
327-
$pulls = array();
328-
$page = 0;
329-
330-
do
327+
catch (\DomainException $e)
331328
{
332-
$page++;
333-
334-
try
335-
{
336-
$items = $github->pulls->getList($this->getState()->get('github_user'), $this->getState()->get('github_repo'), 'open', $page, 100);
337-
}
338-
catch (\DomainException $e)
339-
{
340-
throw new \RuntimeException(\JText::sprintf('COM_PATCHTESTER_ERROR_GITHUB_FETCH', $e->getMessage()));
341-
}
342-
343-
$count = is_array($items) ? count($items) : 0;
344-
345-
if ($count)
346-
{
347-
$pulls = array_merge($pulls, $items);
348-
}
329+
throw new \RuntimeException(\JText::sprintf('COM_PATCHTESTER_ERROR_GITHUB_FETCH', $e->getMessage()));
349330
}
350-
while ($count);
351331

352-
// Dump the old data now
353-
$this->getDb()->truncateTable('#__patchtester_pulls');
332+
$count = is_array($pulls) ? count($pulls) : 0;
354333

355-
// If there are no pulls to insert then bail
356-
if (empty($pulls))
334+
// If there are no pulls to insert then bail, assume we're finished
335+
if ($count === 0 || empty($pulls))
357336
{
358-
return;
337+
return array('complete' => true);
359338
}
360339

361340
$data = array();
@@ -382,6 +361,9 @@ public function requestFromGithub()
382361
{
383362
throw new \RuntimeException(\JText::sprintf('COM_PATCHTESTER_ERROR_INSERT_DATABASE', $e->getMessage()));
384363
}
364+
365+
// Need to make another request
366+
return array('complete' => false, 'page' => ($page + 1));
385367
}
386368

387369
/**

‎administrator/components/com_patchtester/PatchTester/View/Fetch/tmpl/default.php‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
<?php
2+
/**
3+
* Patch testing component for the Joomla! CMS
4+
*
5+
* @copyright Copyright (C) 2011 - 2012 Ian MacLennan, Copyright (C) 2013 - 2015 Open Source Matters, Inc. All rights reserved.
6+
* @license GNU General Public License version 2 or later
7+
*/
8+
9+
/** @type \PatchTester\View\DefaultHtmlView $this */
10+
11+
JHtml::_('jquery.framework');
12+
JHtml::_('script', 'com_patchtester/fetcher.js', false, true);
13+
14+
?>
15+
16+
<div id="patchtester-container">
17+
<h1 id="patchtester-progress-header"><?php echo JText::_('COM_PATCHTESTER_FETCH_INITIALIZING'); ?></h1>
18+
<p id="patchtester-progress-message"><?php echo JText::_('COM_PATCHTESTER_FETCH_INITIALIZING_DESCRIPTION'); ?></p>
19+
<input id="patchtester-token" type="hidden" name="<?php echo JFactory::getSession()->getFormToken(); ?>" value="1" />
20+
</div>

‎administrator/components/com_patchtester/PatchTester/View/Pulls/PullsHtmlView.php‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,19 @@ protected function addToolbar()
102102

103103
if (!count($this->envErrors))
104104
{
105-
\JToolbarHelper::custom('fetch', 'refresh.png', 'refresh_f2.png', 'COM_PATCHTESTER_TOOLBAR_FETCH_DATA', false);
105+
$toolbar = \JToolbar::getInstance('toolbar');
106+
$toolbar->appendButton(
107+
'Popup',
108+
'refresh',
109+
'COM_PATCHTESTER_TOOLBAR_FETCH_DATA',
110+
'index.php?option=com_patchtester&view=fetch&tmpl=component',
111+
500,
112+
210,
113+
0,
114+
0,
115+
'window.parent.location.reload()',
116+
'COM_PATCHTESTER_HEADING_FETCH_DATA'
117+
);
106118
}
107119

108120
\JToolBarHelper::preferences('com_patchtester');

‎administrator/components/com_patchtester/language/en-GB/en-GB.com_patchtester.ini‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,13 @@ COM_PATCHTESTER_CONFLICT_S="The patch could not be applied because it conflicts
1616
COM_PATCHTESTER_ERROR_APPLIED_PATCHES="Cannot fetch data from GitHub while there are applied patches. Please revert those patches before continuing."
1717
COM_PATCHTESTER_ERROR_GITHUB_FETCH="Error retrieving pull requests from GitHub: %s"
1818
COM_PATCHTESTER_ERROR_INSERT_DATABASE="Error inserting pull request data into the database: %s"
19-
COM_PATCHTESTER_FETCH_SUCCESSFUL="Successfully retrieved pull requests."
19+
COM_PATCHTESTER_FETCH_AN_ERROR_HAS_OCCURRED="An error has occurred while fetching the data from GitHub."
20+
COM_PATCHTESTER_FETCH_COMPLETE_CLOSE_WINDOW="All data has been retrieved. Please close this modal window to refresh the page."
21+
COM_PATCHTESTER_FETCH_INITIALIZING="Preparing to fetch GitHub data"
22+
COM_PATCHTESTER_FETCH_INITIALIZING_DESCRIPTION="Making sure all is well to fetch data. Sit tight."
23+
COM_PATCHTESTER_FETCH_PAGE_NUMBER="Processing page %s of GitHub data"
24+
COM_PATCHTESTER_FETCH_PROCESSING="Processing data from GitHub"
25+
COM_PATCHTESTER_FETCH_SUCCESSFUL="Successfully retrieved pull requests"
2026
COM_PATCHTESTER_FIELD_GH_PASSWORD_LABEL="GitHub Account Password"
2127
COM_PATCHTESTER_FIELD_GH_PASSWORD_DESC="Password for the account entered in the "_QQ_"GitHub Account"_QQ_" field. Note that accounts using Two Factor Authentication will not work with this component."
2228
COM_PATCHTESTER_FIELD_GH_TOKEN_DESC="Use this field to input a GitHub API Token in place of your username and password. Note that this is required if your account has Two Factor Authentication enabled."
@@ -31,6 +37,7 @@ COM_PATCHTESTER_FILE_DELETED_DOES_NOT_EXIST_S="The file marked for deletion does
3137
COM_PATCHTESTER_FILE_MODIFIED_DOES_NOT_EXIST_S="The file marked for modification does not exist: %s"
3238
COM_PATCHTESTER_FILTER_APPLIED_PATCHES="Filter Applied Patches"
3339
COM_PATCHTESTER_FILTER_SEARCH_DESCRIPTION="Filter the list by title or ID."
40+
COM_PATCHTESTER_HEADING_FETCH_DATA="Fetching GitHub Data"
3441
COM_PATCHTESTER_PULL_ID="Pull ID"
3542
COM_PATCHTESTER_NO_CREDENTIALS="No user credentials are saved, this will allow only 60 requests to the GitHub API per hour. Saving user credentials will allow 5,000 requests per hour."
3643
COM_PATCHTESTER_NO_ITEMS="No data has been retrieved from GitHub, please click the 'Fetch Data' button in the toolbar to retrieve the open pull requests."

‎administrator/components/com_patchtester/patchtester.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
<folder>css</folder>
3636
<folder>fonts</folder>
3737
<folder>images</folder>
38+
<folder>js</folder>
3839
</media>
3940
<administration>
4041
<menu img="../media/com_patchtester/images/icon-16-patchtester.png">com_patchtester</menu>

0 commit comments

Comments
 (0)