small code quality fixups

Author: ptooley

pypop/dimemas.py

@@ -94,11 +94,10 @@ def dimemas_analyse(tracefile, configfile, outpath=None, substrings=None):
 
     # Make sure config._tmpdir_path exists before using it
     if config._tmpdir_path:
-        try:
-            os.makedirs(config._tmpdir_path, exist_ok=True)
-        except OSError as err:
-            print("FATAL: {}".format(err))
-    workdir = mkdtemp(dir=config._tmpdir_path)
+        os.makedirs(config._tmpdir_path, exist_ok=True)
+        workdir = mkdtemp(dir=config._tmpdir_path)
+    else:
+        workdir = mkdtemp()
 
     # Create temporary config from supplied config and substitution dict
     dimconfig = os.path.join(workdir, ".tmpconfig".join(splitext(basename(configfile))))

pypop/extrae.py

@@ -14,6 +14,7 @@
 from os.path import basename, dirname, normpath, splitext
 from tempfile import mkdtemp, mkstemp
 import subprocess as sp
+import warnings
 
 from pkg_resources import resource_filename
 
@@ -23,6 +24,8 @@
 from .prv import PRV, get_prv_header_info
 from . import config
 
+from pypop.utils.exceptions import ExtraePRVNoOnOffEventsError
+
 floatmatch = re.compile(r"[0-9,]+\.[0-9]+")
 keymatch = re.compile(r"@.*@")
 tabmatch = re.compile(r"\t+")
@@ -111,13 +114,14 @@ def chop_prv_to_roi(prv_file, outfile=None):
     if outfile:
         workdir = dirname(normpath(outfile))
     else:
-        tgtname = ".chop".join(splitext(basename(prv_file)))
+        if prv_file.endswith(".gz"):
+            tgtname = ".chop".join(splitext(basename(prv_file[:-3])))
+        else:
+            tgtname = ".chop".join(splitext(basename(prv_file)))
+
         # Make sure config._tmpdir_path exists before using it
         if config._tmpdir_path:
-            try:
-                os.makedirs(config._tmpdir_path, exist_ok=True)
-            except OSError as err:
-                print("FATAL: {}".format(err))
+            os.makedirs(config._tmpdir_path, exist_ok=True)
         workdir = mkdtemp(dir=config._tmpdir_path)
         outfile = os.path.join(workdir, tgtname)
 
@@ -189,7 +193,7 @@ def chop_prv_to_roi(prv_file, outfile=None):
 
 
 def _get_roi_times(roi_prv):
-    """ Extract ROi timing information from a filtered trace
+    """ Extract ROI timing information from a filtered trace
 
     Expects a trace containing only Extrae On/Off events and returns tuple of
     earliest and latest time
@@ -211,7 +215,7 @@ def _get_roi_times(roi_prv):
     ontime, offtime = (ons["time"].min(), 1 + offs["time"].max())
 
     if ontime is numpy.nan or offtime is numpy.nan:
-        raise ValueError("Unable to locate valid ON-OFF bracket in trace")
+        raise ExtraePRVNoOnOffEventsError("No valid Extrae ON-OFF bracket in trace")
 
     return ontime, offtime
 
@@ -389,12 +393,10 @@ def run_paramedir(tracefile, paramedir_config, outfile=None, variables=None):
 
     # Make sure config._tmpdir_path exists before using it
     if config._tmpdir_path:
-        try:
-            os.makedirs(config._tmpdir_path, exist_ok=True)
-        except OSError as err:
-            print("FATAL: {}".format(err))
-
-    tmpdir = mkdtemp(dir=config._tmpdir_path)
+        os.makedirs(config._tmpdir_path, exist_ok=True)
+        tmpdir = mkdtemp(dir=config._tmpdir_path)
+    else:
+        tmpdir = mkdtemp()
 
     # If variables is none, still sub with empty dict
     variables = variables if variables else {}
@@ -476,7 +478,7 @@ def _split_countline(countline, bins):
     extra_strings = len(count_strings) - num_values
 
     if extra_strings > 1:
-        print(
+        warnings.warn(
             "Warning, got more label strings ({}) than expected (1)"
             "".format(extra_strings)
         )

pypop/notebook_interface/fileselector.py

@@ -2,8 +2,10 @@
 # SPDX-License-Identifier: BSD-3-Clause-Clear
 # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved.
 
+from os import getcwd
 from os.path import expanduser, normpath
 
+from ipyfilechooser import FileChooser
 from ipywidgets import (
     Accordion,
     Box,
@@ -15,21 +17,26 @@
     Layout,
     Label,
 )
-from ipyfilechooser import FileChooser
 
 
 class ValidatingChooser(VBox):
 
     _container_layout = Layout(margin="1em 0em 1em 0em")
     _msgbox_layout = Layout()
 
-    def __init__(self, starting_file=None, **kwargs):
+    def __init__(self, starting_file=None, starting_path=None, **kwargs):
 
         if starting_file is None:
             starting_file = ""
 
+        if starting_path is None:
+            starting_path = getcwd()
+
         self._filechooser = FileChooser(
-            filename=starting_file, select_default=bool(starting_file),
+            filename=starting_file,
+            path=starting_path,
+            select_default=bool(starting_file),
+            use_dir_icons=True,
         )
         self._msgbox = HBox(children=[], layout=self._msgbox_layout)
         self._validity = None
@@ -129,6 +136,10 @@ def _create_buttons(self):
         )
 
     def _create_advanced_config_box(self):
+        self._advanced_config_controls["Chop to ROI"] = Checkbox(
+            value=True, description="Chop to ROI"
+        )
+
         self._advanced_config_controls["Delete Cache"] = Checkbox(
             value=False, description="Delete Cache"
         )
@@ -148,7 +159,9 @@ def _add_filechooser_row(self, callback_reference=None):
     def _create_filechoosers(self):
 
         self._filechoosers = [
-            ValidatingChooser(starting_file=fname) if fname else ValidatingChooser()
+            ValidatingChooser(starting_file=fname)
+            if fname
+            else ValidatingChooser(starting_path=self._base_dir)
             for fname in self._files
         ]
         [fc.set_file_select_callback(self._update_files) for fc in self._filechoosers]

pypop/notebook_interface/wizard.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: BSD-3-Clause-Clear
 # Copyright (c) 2019, The Numerical Algorithms Group, Ltd. All rights reserved.
 
-from ipywidgets import Tab, Layout
+from ipywidgets import Tab, Layout, VBox, Text
 
 import pypop.metrics.metricset as pm
 from ..traceset import TraceSet
@@ -11,6 +11,7 @@
 from .bokeh_widget import BokehWidgetWrapper
 from .reporting import ReportGenerator
 
+from pypop.utils.exceptions import ExtraePRVNoOnOffEventsError
 
 class TypeAsserter:
 
@@ -113,18 +114,21 @@ def __init__(self, metric_calc="auto", base_dir=".", starting_files=None, **kwar
         self._scaling_plot = None
         self._report_generator = None
         self._metric_calculator = metric_calc
+        self._base_dir = base_dir
 
         self._analysis_state = AnalysisState()
 
         self._fileselector = FileSelector(
-            base_dir=base_dir,
+            base_dir=self._base_dir,
             starting_files=starting_files,
             calculation_callback=self._calculate_callback_hook,
             analysis_state=self._analysis_state,
         )
 
+        self._status_box = VBox()
+
         super().__init__(
-            children=[self._fileselector],
+            children=[VBox([self._status_box, self._fileselector])],
             layout=Layout(width="auto", max_width="1280px"),
             **kwargs
         )
@@ -133,9 +137,23 @@ def __init__(self, metric_calc="auto", base_dir=".", starting_files=None, **kwar
 
     def _calculate_callback_hook(self, callback_reference=None):
 
-        statistics = TraceSet(
-            self._fileselector.filenames, force_recalculation=False, chop_to_roi=True
-        )
+        advanced_config = self._fileselector._advanced_config_controls
+
+        try:
+            statistics = TraceSet(
+                self._fileselector.filenames,
+                force_recalculation=advanced_config["Delete Cache"].value,
+                chop_to_roi=advanced_config["Chop to ROI"].value
+            )
+        except ExtraePRVNoOnOffEventsError as err:
+            warnstr = "Warning: Disabling Chopping to ROI ({})".format(err)
+            self._status_box.children = [Text(warnstr, layout=Layout(width='auto'))]
+            statistics = TraceSet(
+                self._fileselector.filenames,
+                force_recalculation=advanced_config["Delete Cache"].value,
+                chop_to_roi=False,
+            )
+
         if self._metric_calculator in ("auto", None):
             self._metric_calculate = statistics.suggested_metrics
 

pypop/prv.py

@@ -85,21 +85,24 @@ class PRV:
 
     def __init__(self, prv_path):
 
-        if prv_path and prv_path.endswith((".prv", ".prv.gz")):
-            self._event_names = {}
-            self._event_vals = {}
-            self._parse_pcf(prv_path)
-
-            self._parse_prv(prv_path)
+        try:
+            self._load_prv_and_pcf(prv_path)
 
-        else:
+        except ValueError:
             try:
                 self._load_pickle(prv_path)
             except ValueError:
                 raise ValueError("Not a prv or valid pickle")
 
         self._omp_region_data = None
 
+    def _load_prv_and_pcf(self, prv_path):
+        self._event_names = {}
+        self._event_vals = {}
+        self._parse_pcf(prv_path)
+
+        self._parse_prv(prv_path)
+
     def _parse_pcf(self, prv_path):
 
         if prv_path.endswith(".gz"):
@@ -199,10 +202,16 @@ def save(self, filename):
             pickle.dump(savedata, fh)
 
     def _load_pickle(self, filename):
-        with gzip.open(filename, "rb") as fh:
-            data = pickle.load(fh)
-
         try:
+            with gzip.open(filename, "rb") as fh:
+                data = pickle.load(fh)
+        except gzip.BadGzipFile:
+            try:
+                with open(filename, "rb") as fh:
+                    data = pickle.load(fh)
+            except pickle.UnpicklingError:
+                raise ValueError("Invalid pickle -- missing data")
+
             self.metadata, self.state, self.event, self.comm = data
         except ValueError:
             raise ValueError("Invalid pickle -- missing data")
@@ -543,7 +552,7 @@ def _parse_paraver_headerline(headerline):
         _format_timing(elems[1]),
         *_split_nodestring(elems[2]),
         _format_num_apps(elems[3]),
-        _format_app_list(elems[4])
+        _format_app_list(elems[4]),
     )
 
     return metadata

pypop/trace/prvtrace.py

@@ -144,18 +144,15 @@ def _split_layoutstring(prv_td):
     @staticmethod
     def _analyze_tracefile(trace, chop_to_roi, outpath):
         if outpath:
-            try:
-                makedirs(outpath, exist_ok=True)
-            except OSError as err:
-                print("Fatal: {}".format(err))
+            makedirs(outpath, exist_ok=True)
 
         if chop_to_roi:
             if outpath:
                 tgtname = ".chop".join(splitext(basename(trace)))
                 outfile = path_join(outpath, tgtname)
             else:
                 outfile = None
-            cut_trace = chop_prv_to_roi(trace, outfile)
+                cut_trace = chop_prv_to_roi(trace, outfile)
         else:
             cut_trace = trace
 

pypop/utils/exceptions.py

@@ -7,3 +7,7 @@ class WrongLoaderError(RuntimeError):
 
 class UnknownLoaderError(RuntimeError):
     pass
+
+
+class ExtraePRVNoOnOffEventsError(RuntimeError):
+    pass