remove inverted index feature

Author: yooper

.travis.yml

@@ -1,9 +1,8 @@
 language: php
 
 php:
-  - 5.5
-  - 5.6
   - 7.0
+  - 7.1
   - hhvm
 
 before_script:

README.md

@@ -16,25 +16,36 @@ composer require yooper/php-text-analysis
 Documentation for the library resides in the wiki. 
 https://github.com/yooper/php-text-analysis/wiki
 
+
+
+
 Dictionary Installation
 =============
 
 Not required unless you use the dictionary stemmers
 
-*For Ubuntu*
+*For Ubuntu < 16*
 ```
 sudo apt-get install libpspell-dev 
 sudo apt-get install php5-pspell
 sudo apt-get install aspell-en
 sudo apt-get install php5-enchant
 ```
+*For Ubuntu >= 16*
+```
+sudo apt-get install libpspell-dev php7.0-pspell aspell-en php7.0-enchant
+```
+
+
 *For Centos* 
 ```
 sudo yum install php5-pspell
 sudo yum install aspell-en
 sudo yum install php5-enchant
 ```
 
+*PHP Pecl Stem* is not currently available in php 7.0. 
+
 
 Tokenize
 =============

composer.json

@@ -29,7 +29,7 @@
         "symfony/console": ">=2.7"
     },
     "require-dev": {
-        "phpunit/phpunit": "4.*",
-        "mockery/mockery" : "0.9.4"
+        "phpunit/phpunit": "5.*",
+        "mockery/mockery" : "0.9.7"
     }
 }

src/Console/Commands/NltkPackageInstallCommand.php

@@ -6,18 +6,30 @@
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Helper\ProgressBar;
 
 use TextAnalysis\Downloaders\DownloadPackageFactory as DPF;
 use TextAnalysis\Downloaders\NltkCorporaIndexDownloader;
 
 
+
 /**
  * Installs the selected nltk corpus package 
  *
  * @author yooper
  */
 class NltkPackageInstallCommand extends Command
 {
+    /**
+    * @var ProgressBar
+    */
+    protected $progressBar = null;
+    
+    /**
+     * @var \Symfony\Component\Console\Output\OutputInterface
+     */
+    private $output;
+
     protected function configure()
     {
         $this->setName('pta:install:package')
@@ -31,25 +43,88 @@ protected function configure()
 
     protected function execute(InputInterface $input, OutputInterface $output)
     {        
+        $this->output = $output;
         $packageId = $input->getArgument('package');
 
         $listPackages = (new NltkCorporaIndexDownloader())->getPackages();
 
-        $packageFound = false;
+        $packageFound = null;
 
         foreach($listPackages as $package) 
         { 
             if($packageId == $package->getId()) {
-                $packageFound = true;
-                $download = DPF::download($package);                
+                $packageFound = $package;
                 break;
             }
         }
 
         if(!$packageFound) { 
             $output->writeln("Package {$packageId} was not found, try textconsole pta:list, to see the available packages");
         } else {
+            
+            $download = DPF::download($package);  
+            // Create stream context.
+            $context = stream_context_create([], ['notification' => [$this, 'progress']]);
+
+            // Pipe file.
+            $resource = fopen($packageFound->getUrl(), 'r', null, $context);
+            $stream = fopen($download->getDownloadFullPath(), 'w+');
+            if (!$stream) {
+                $output->writeln("Package {$packageFound->getId()} - {$packageFound->getName()} install failed, permission denied to create file into {$download->getDownloadFullPath()}");
+            }
+
+            stream_copy_to_stream($resource, $stream);
+            
+            if (!fclose($stream)) {
+                $output->writeln("Could not save file {$download->getDownloadFullPath()}");
+            }
+
+            // End output.
+            $this->progressBar->finish();
+
+            if(!$download->verifyChecksum()) {
+                $output->writeln("Bad checksum for the downloaded package {$packageFound->getId()}");
+                exit;        
+            }
+            $download->unpackPackage();
+            $output->writeln(PHP_EOL);
             $output->writeln("Package {$package->getId()} - {$package->getName()} was installed into {$download->getInstallDir()}");
         }        
     }
+
+    /**
+     * @param int $notificationCode
+     * @param int $severity
+     * @param string $message
+     * @param int $messageCode
+     * @param int $bytesTransferred
+     * @param int $bytesMax
+     */
+    public function progress($notificationCode, $severity, $message, $messageCode, $bytesTransferred, $bytesMax)
+    {
+        if (STREAM_NOTIFY_REDIRECTED === $notificationCode) {
+            $this->progressBar->clear();
+            $this->progressBar = null;
+            return;
+        }
+
+        if (STREAM_NOTIFY_FILE_SIZE_IS === $notificationCode) {
+            if ($this->progressBar) {
+                $this->progressBar->clear();
+            }
+            $this->progressBar = new ProgressBar($this->output, $bytesMax);
+        }
+
+        if (STREAM_NOTIFY_PROGRESS === $notificationCode) {
+            if (is_null($this->progressBar)) {
+                $this->progressBar = new ProgressBar($this->output);
+            }
+            $this->progressBar->setProgress($bytesTransferred);
+        }
+
+        if (STREAM_NOTIFY_COMPLETED === $notificationCode) {
+            $this->finish($bytesTransferred);
+        }
+    }
+
 }

src/Console/Commands/StemmerExceptionListCommand.php

@@ -31,14 +31,7 @@ protected function __construct(Package $package)
         if(file_exists($this->getDownloadFullPath()) && $this->verifyChecksum()) {
             return;
         }     
-        
-        $this->downloadRemoteFile();
-        
-        if($this->verifyChecksum()) {
-            throw new Exception("Bad checksum for the downloaded package {$this->getPackage()->getId()}");            
-        }
-        
-        $this->unpackPackage();        
+               
     }
 
     /**
@@ -52,22 +45,24 @@ static public function download(Package $package)
     }
 
     /**
-     * Verify the packages checksum against the downloaded file, if it exists
+     * Verify the packages checksum against the downloaded file
+     * if the package has a checksum
      * @return boolean
      */
     public function verifyChecksum()
     {
         if(empty($this->getPackage()->getChecksum())) {
             return true;
         }
+        
         return $this->getPackage()->getChecksum() === md5($this->getDownloadFullPath());
     }
 
     /**
      * de-compress the downloaded corpus into the install directory, or
      * copy the files into the install directory
      */
-    protected function unpackPackage()
+    public function unpackPackage()
     {
         // it is zipped, we must unzip it
         if($this->getPackage()->getUnzip()) {
@@ -150,21 +145,6 @@ public function initialize()
 
     }
 
-    /**
-     * @todo improve downloader code, make it more robust
-     */
-    protected function downloadRemoteFile()
-    {
-        $handle = fopen($this->getPackage()->getUrl(), "rb");
-        $fp = fopen($this->getDownloadFullPath(), 'w');
-        $content = '';
-        while (!feof($handle)) {
-          $content = fread($handle, 8192);
-          fwrite($fp, $content);
-        }
-        fclose($handle);
-        fclose($fp);
-    }
 
     /**
      * Has the full path to where the download should go

src/Downloaders/DownloadPackageFactory.php

@@ -52,6 +52,10 @@ public function getPackages()
             {
                 $data = (array)$package;
                 extract($data['@attributes']);
+                // checksums may not exist on some remote packages
+                if(!isset($checksum)) {
+                    $checksum = null;
+                }                
                 $this->packages[] = new Package($id, $checksum, $name, $subdir, $unzip, $url);
             }            
         }