From cc3ce675f03e9f8478092058a603588aa28cbb18 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Fri, 21 Oct 2022 09:42:30 +0200 Subject: [PATCH 01/26] improve get_io mechansim to work with folders and add tests --- neo/io/__init__.py | 53 ++++++++++++++++++++++++++++--- neo/rawio/__init__.py | 2 +- neo/rawio/alphaomegarawio.py | 3 ++ neo/test/iotest/common_io_test.py | 8 +++++ 4 files changed, 60 insertions(+), 6 deletions(-) diff --git a/neo/io/__init__.py b/neo/io/__init__.py index 0c8697cb9..b787f63e3 100644 --- a/neo/io/__init__.py +++ b/neo/io/__init__.py @@ -262,7 +262,8 @@ """ -import os.path +import pathlib +from collections import Counter # try to import the neuroshare library. # if it is present, use the neuroshareapiio to load neuroshare files @@ -384,14 +385,56 @@ WinWcpIO ] +# for each supported extension list the ios supporting it +io_by_extension = {} +for io in iolist: + for extension in io.extensions: + # extension handling should not be case sensitive + io_by_extension.setdefault(extension, []).append(io) -def get_io(filename, *args, **kwargs): + +def get_io(file_or_folder, *args, **kwargs): """ Return a Neo IO instance, guessing the type based on the filename suffix. """ - extension = os.path.splitext(filename)[1][1:] + file_or_folder = pathlib.Path(file_or_folder) + + if file_or_folder.is_file(): + return get_io_from_filename(file_or_folder, *args, **kwargs) + + elif file_or_folder.is_dir(): + # find the io that fits the best with the files contained in the folder + potential_ios = [] + + # scan files in folder to determine io type + filenames = [f for f in file_or_folder.glob('*') if f.is_file()] + for filename in filenames: + extension = filename.suffix[1:].lower() + for io in iolist: + if extension in io.extensions: + potential_ios.append(io) + + if potential_ios: + most_common_io = Counter(potential_ios).most_common()[0][0] + return most_common_io(file_or_folder, *args, **kwargs) + + raise IOError(f"Could not identify IO for {file_or_folder}") + + +def get_io_from_filename(filename, *args, **kwargs): + """ + Return a Neo IO instance, guessing the type based on the filename suffix. + """ + extension = pathlib.Path(filename).suffix[1:].lower() + + + print(extension) for io in iolist: if extension in io.extensions: - return io(filename, *args, **kwargs) + # some extensions are used by multiple systems. + try: + return io(filename, *args, **kwargs) + except: + continue - raise IOError("File extension %s not registered" % extension) + raise IOError(f"Could not identify IO for {filename}") diff --git a/neo/rawio/__init__.py b/neo/rawio/__init__.py index 9dbc0ff95..85b57b3ff 100644 --- a/neo/rawio/__init__.py +++ b/neo/rawio/__init__.py @@ -235,7 +235,7 @@ def get_rawio_class(filename_or_dirname): """ - Return a neo.rawio class guess from file extention. + Return a neo.rawio class guess from file extension. """ _, ext = os.path.splitext(filename_or_dirname) ext = ext[1:] diff --git a/neo/rawio/alphaomegarawio.py b/neo/rawio/alphaomegarawio.py index 9ad0b8b16..13fe24d00 100644 --- a/neo/rawio/alphaomegarawio.py +++ b/neo/rawio/alphaomegarawio.py @@ -90,6 +90,9 @@ class AlphaOmegaRawIO(BaseRawIO): def __init__(self, dirname="", lsx_files=None, prune_channels=True): super().__init__(dirname=dirname) self.dirname = Path(dirname) + if not self.dirname.is_dir(): + raise ValueError(f'{dirname} is not a directory.') + self._lsx_files = lsx_files self._mpx_files = None if self.dirname.is_dir(): diff --git a/neo/test/iotest/common_io_test.py b/neo/test/iotest/common_io_test.py index 979e42f4a..085b25b2b 100644 --- a/neo/test/iotest/common_io_test.py +++ b/neo/test/iotest/common_io_test.py @@ -34,6 +34,7 @@ from neo.test.rawiotest.common_rawio_test import repo_for_test from neo.utils import (download_dataset, get_local_testing_data_folder) +from neo import get_io try: import datalad @@ -528,3 +529,10 @@ def test__handle_pathlib_filename(self): self.ioclass(filename=pathlib_filename) elif self.ioclass.mode == 'dir': self.ioclass(dirname=pathlib_filename) + + def test_get_io(self): + for entity in self.entities_to_test: + entity = get_test_file_full_path(self.ioclass, filename=entity, + directory=self.local_test_dir) + io = get_io(entity) + assert isinstance(io, self.ioclass) From cbc83c1850a85d49a502d364c183f5686c55d5fb Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Mon, 24 Oct 2022 16:39:00 +0200 Subject: [PATCH 02/26] provide function to list all potentially relevant ios instead of returning a single one. --- neo/io/__init__.py | 72 +++++++++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 27 deletions(-) diff --git a/neo/io/__init__.py b/neo/io/__init__.py index b787f63e3..ef5730c78 100644 --- a/neo/io/__init__.py +++ b/neo/io/__init__.py @@ -389,6 +389,7 @@ io_by_extension = {} for io in iolist: for extension in io.extensions: + extension = extension.lower() # extension handling should not be case sensitive io_by_extension.setdefault(extension, []).append(io) @@ -397,44 +398,61 @@ def get_io(file_or_folder, *args, **kwargs): """ Return a Neo IO instance, guessing the type based on the filename suffix. """ + ios = list_candidate_ios(file_or_folder) + for io in ios: + try: + return io(file_or_folder, *args, **kwargs) + except: + continue + + raise IOError(f"Could not identify IO for {file_or_folder}") + + +def list_candidate_ios(file_or_folder): + """ + Identify neo IO that can potentially load data in the file or folder + + Parameters + ---------- + file_or_folder (str, pathlib.Path) + Path to the file or folder to load + + Returns + ------- + list + List of neo io classes that are associated with the file extensions detected + """ file_or_folder = pathlib.Path(file_or_folder) if file_or_folder.is_file(): - return get_io_from_filename(file_or_folder, *args, **kwargs) + suffix = file_or_folder.suffix[1:].lower() + if suffix not in io_by_extension: + print(f'{suffix} not found') + return io_by_extension[suffix] elif file_or_folder.is_dir(): - # find the io that fits the best with the files contained in the folder - potential_ios = [] - # scan files in folder to determine io type filenames = [f for f in file_or_folder.glob('*') if f.is_file()] - for filename in filenames: - extension = filename.suffix[1:].lower() - for io in iolist: - if extension in io.extensions: - potential_ios.append(io) - if potential_ios: - most_common_io = Counter(potential_ios).most_common()[0][0] - return most_common_io(file_or_folder, *args, **kwargs) + # if only file prefix was provided, e.g /mydatafolder/session1- + # to select all files sharing the `session1-` prefix + elif file_or_folder.parent.exists(): + filenames = file_or_folder.parent.glob(file_or_folder.name + '*') - raise IOError(f"Could not identify IO for {file_or_folder}") + # find the io that fits the best with the files contained in the folder + potential_ios = [] + for filename in filenames: + for suffix in filename.suffixes: + suffix = suffix[1:].lower() + if suffix in io_by_extension: + potential_ios.extend(io_by_extension[suffix]) + if not potential_ios: + raise ValueError(f'Could not determine io to load {file_or_folder}') -def get_io_from_filename(filename, *args, **kwargs): - """ - Return a Neo IO instance, guessing the type based on the filename suffix. - """ - extension = pathlib.Path(filename).suffix[1:].lower() + # return ios ordered by number of files supported + counter = Counter(potential_ios).most_common() + return [io for io, count in counter] - print(extension) - for io in iolist: - if extension in io.extensions: - # some extensions are used by multiple systems. - try: - return io(filename, *args, **kwargs) - except: - continue - raise IOError(f"Could not identify IO for {filename}") From 676575c419c6c7b5fcefef251c3dfd50bca53a86 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Mon, 24 Oct 2022 16:40:12 +0200 Subject: [PATCH 03/26] update ios to list all format related extensions (even if not used by io) --- neo/io/__init__.py | 2 +- neo/rawio/axographrawio.py | 2 +- neo/rawio/biocamrawio.py | 2 +- neo/rawio/blackrockrawio.py | 5 ++++- neo/rawio/neuralynxrawio/neuralynxrawio.py | 2 +- neo/rawio/nixrawio.py | 2 +- neo/rawio/openephysrawio.py | 3 ++- neo/rawio/phyrawio.py | 3 ++- neo/rawio/spikeglxrawio.py | 3 ++- neo/rawio/tdtrawio.py | 1 + 10 files changed, 16 insertions(+), 9 deletions(-) diff --git a/neo/io/__init__.py b/neo/io/__init__.py index ef5730c78..ffd6860b4 100644 --- a/neo/io/__init__.py +++ b/neo/io/__init__.py @@ -351,7 +351,7 @@ CedIO, EDFIO, ElanIO, - # ElphyIO, + ElphyIO, ExampleIO, IgorIO, IntanIO, diff --git a/neo/rawio/axographrawio.py b/neo/rawio/axographrawio.py index 975637bcf..cb58e314c 100644 --- a/neo/rawio/axographrawio.py +++ b/neo/rawio/axographrawio.py @@ -218,7 +218,7 @@ class AxographRawIO(BaseRawIO): """ name = 'AxographRawIO' description = 'This IO reads .axgd/.axgx files created with AxoGraph' - extensions = ['axgd', 'axgx'] + extensions = ['axgd', 'axgx', ''] rawmode = 'one-file' def __init__(self, filename, force_single_segment=False): diff --git a/neo/rawio/biocamrawio.py b/neo/rawio/biocamrawio.py index 9b9284e63..d61d92dae 100644 --- a/neo/rawio/biocamrawio.py +++ b/neo/rawio/biocamrawio.py @@ -29,7 +29,7 @@ class BiocamRawIO(BaseRawIO): >>> float_chunk = r.rescale_signal_raw_to_float(raw_chunk, dtype='float64', channel_indexes=[0, 3, 6]) """ - extensions = ['h5'] + extensions = ['h5', 'brw'] rawmode = 'one-file' def __init__(self, filename=''): diff --git a/neo/rawio/blackrockrawio.py b/neo/rawio/blackrockrawio.py index 7b2d4e7e0..9356dfb73 100644 --- a/neo/rawio/blackrockrawio.py +++ b/neo/rawio/blackrockrawio.py @@ -122,7 +122,7 @@ class BlackrockRawIO(BaseRawIO): """ extensions = ['ns' + str(_) for _ in range(1, 7)] - extensions.extend(['nev', ]) # 'sif', 'ccf' not yet supported + extensions.extend(['nev', 'sif', 'ccf']) # 'sif', 'ccf' not yet supported rawmode = 'multi-file' def __init__(self, filename=None, nsx_override=None, nev_override=None, @@ -154,6 +154,9 @@ def __init__(self, filename=None, nsx_override=None, nev_override=None, else: self._filenames['nev'] = self.filename + self._filenames['sif'] = self.filename + self._filenames['ccf'] = self.filename + # check which files are available self._avail_files = dict.fromkeys(self.extensions, False) self._avail_nsx = [] diff --git a/neo/rawio/neuralynxrawio/neuralynxrawio.py b/neo/rawio/neuralynxrawio/neuralynxrawio.py index cc8c812ac..8f46bb9d0 100644 --- a/neo/rawio/neuralynxrawio/neuralynxrawio.py +++ b/neo/rawio/neuralynxrawio/neuralynxrawio.py @@ -75,7 +75,7 @@ class NeuralynxRawIO(BaseRawIO): Display all information about signal channels, units, segment size.... """ - extensions = ['nse', 'ncs', 'nev', 'ntt'] + extensions = ['nse', 'ncs', 'nev', 'ntt', 'nvt', 'nrd'] # nvt and nrd are not yet supported rawmode = 'one-dir' _ncs_dtype = [('timestamp', 'uint64'), ('channel_id', 'uint32'), ('sample_rate', 'uint32'), diff --git a/neo/rawio/nixrawio.py b/neo/rawio/nixrawio.py index f5f097dd9..21f47cb24 100644 --- a/neo/rawio/nixrawio.py +++ b/neo/rawio/nixrawio.py @@ -31,7 +31,7 @@ class NIXRawIO(BaseRawIO): - extensions = ['nix'] + extensions = ['nix', 'h5'] rawmode = 'one-file' def __init__(self, filename=''): diff --git a/neo/rawio/openephysrawio.py b/neo/rawio/openephysrawio.py index 9026fd128..f2a3aa0ff 100644 --- a/neo/rawio/openephysrawio.py +++ b/neo/rawio/openephysrawio.py @@ -64,7 +64,8 @@ class OpenEphysRawIO(BaseRawIO): aligned, and a warning is emitted. """ - extensions = [] + # file formats used by openephys + extensions = ['continuous', 'openephys', 'spikes', 'events', 'xml'] rawmode = 'one-dir' def __init__(self, dirname=''): diff --git a/neo/rawio/phyrawio.py b/neo/rawio/phyrawio.py index a2c23503e..027275075 100644 --- a/neo/rawio/phyrawio.py +++ b/neo/rawio/phyrawio.py @@ -33,7 +33,8 @@ class PhyRawIO(BaseRawIO): >>> spike_times = r.rescale_spike_timestamp(spike_timestamp, 'float64') """ - extensions = [] + # file formats used by phy + extensions = ['npy', 'mat', 'tsv', 'dat'] rawmode = 'one-dir' def __init__(self, dirname='', load_amplitudes=False, load_pcs=False): diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py index c66c1ca23..3bececd38 100644 --- a/neo/rawio/spikeglxrawio.py +++ b/neo/rawio/spikeglxrawio.py @@ -64,7 +64,8 @@ class SpikeGLXRawIO(BaseRawIO): load_sync_channel=False/True The last channel (SY0) of each stream is a fake channel used for synchronisation. """ - extensions = [] + # file formats used by spikeglxio + extensions = ['meta', 'bin'] rawmode = 'one-dir' def __init__(self, dirname='', load_sync_channel=False, load_channel_location=False): diff --git a/neo/rawio/tdtrawio.py b/neo/rawio/tdtrawio.py index 5d3f17fcf..7fc0e3b58 100644 --- a/neo/rawio/tdtrawio.py +++ b/neo/rawio/tdtrawio.py @@ -34,6 +34,7 @@ class TdtRawIO(BaseRawIO): + extensions = ['tbk', 'tdx', 'tev', 'tin', 'tnt', 'tsq', 'sev', 'txt'] rawmode = 'one-dir' def __init__(self, dirname='', sortname=''): From 205b44e2bdbad1ec60b3f2045ac5ce4bd76def7c Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Mon, 24 Oct 2022 16:40:30 +0200 Subject: [PATCH 04/26] update tests --- neo/test/iotest/common_io_test.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/neo/test/iotest/common_io_test.py b/neo/test/iotest/common_io_test.py index 085b25b2b..eea419bd6 100644 --- a/neo/test/iotest/common_io_test.py +++ b/neo/test/iotest/common_io_test.py @@ -34,7 +34,7 @@ from neo.test.rawiotest.common_rawio_test import repo_for_test from neo.utils import (download_dataset, get_local_testing_data_folder) -from neo import get_io +from neo import list_candidate_ios try: import datalad @@ -530,9 +530,11 @@ def test__handle_pathlib_filename(self): elif self.ioclass.mode == 'dir': self.ioclass(dirname=pathlib_filename) - def test_get_io(self): + def test_list_candidate_ios(self): for entity in self.entities_to_test: entity = get_test_file_full_path(self.ioclass, filename=entity, directory=self.local_test_dir) - io = get_io(entity) - assert isinstance(io, self.ioclass) + ios = list_candidate_ios(entity) + if self.ioclass not in ios: + print(f'{entity} does not list {self.ioclass} in list of ios: {ios}') + assert self.ioclass in ios From e8c1ff0defb3f1df56263ab0f7f2272cd37c321d Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Mon, 24 Oct 2022 16:43:27 +0200 Subject: [PATCH 05/26] update related file extension --- neo/io/neuroshareapiio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/io/neuroshareapiio.py b/neo/io/neuroshareapiio.py index 3db1b9fd9..86d872230 100644 --- a/neo/io/neuroshareapiio.py +++ b/neo/io/neuroshareapiio.py @@ -72,7 +72,7 @@ class NeuroshareapiIO(BaseIO): name = "Neuroshare" - extensions = [] + extensions = ['mcd'] # This object operates on neuroshare files mode = "file" From 952a783ee744c5ac4f88ef9c902bc32343a3f3ef Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Tue, 25 Oct 2022 18:12:42 +0200 Subject: [PATCH 06/26] [alphaomega] raise error for non-existent data directory --- neo/rawio/alphaomegarawio.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/neo/rawio/alphaomegarawio.py b/neo/rawio/alphaomegarawio.py index 13fe24d00..8f3267e64 100644 --- a/neo/rawio/alphaomegarawio.py +++ b/neo/rawio/alphaomegarawio.py @@ -90,19 +90,18 @@ class AlphaOmegaRawIO(BaseRawIO): def __init__(self, dirname="", lsx_files=None, prune_channels=True): super().__init__(dirname=dirname) self.dirname = Path(dirname) - if not self.dirname.is_dir(): - raise ValueError(f'{dirname} is not a directory.') self._lsx_files = lsx_files self._mpx_files = None - if self.dirname.is_dir(): - self._explore_folder() - else: - self.logger.error(f"{self.dirname} is not a folder") self._prune_channels = prune_channels self._opened_files = {} self._ignore_unknown_datablocks = True # internal debug property + if self.dirname.is_dir(): + self._explore_folder() + else: + raise IOError(f"{self.dirname} is not a folder") + def _explore_folder(self): """ If class was instantiated with lsx_files (list of .lsx files), load only From 043e338e397be25f3366cc6a507725a21f7970a4 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Tue, 25 Oct 2022 18:46:17 +0200 Subject: [PATCH 07/26] do not overwrite module with variable of the same name --- neo/io/__init__.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/neo/io/__init__.py b/neo/io/__init__.py index ffd6860b4..599acfc8b 100644 --- a/neo/io/__init__.py +++ b/neo/io/__init__.py @@ -10,6 +10,7 @@ Functions: .. autofunction:: neo.io.get_io +.. autofunction:: neo.io.list_candidate_ios Classes: @@ -387,11 +388,11 @@ # for each supported extension list the ios supporting it io_by_extension = {} -for io in iolist: - for extension in io.extensions: +for current_io in iolist: # do not use `io` as variable name here as this overwrites the module io + for extension in current_io.extensions: extension = extension.lower() # extension handling should not be case sensitive - io_by_extension.setdefault(extension, []).append(io) + io_by_extension.setdefault(extension, []).append(current_io) def get_io(file_or_folder, *args, **kwargs): @@ -453,6 +454,3 @@ def list_candidate_ios(file_or_folder): # return ios ordered by number of files supported counter = Counter(potential_ios).most_common() return [io for io, count in counter] - - - From 9e00b19e8c0faf2ecef748c5848dac281097a9bc Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Wed, 26 Oct 2022 08:41:46 +0200 Subject: [PATCH 08/26] [alphaomega] update tests to check for non-existent data folder --- neo/rawio/alphaomegarawio.py | 2 +- neo/test/rawiotest/test_alphaomegarawio.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/neo/rawio/alphaomegarawio.py b/neo/rawio/alphaomegarawio.py index 8f3267e64..23de7cac9 100644 --- a/neo/rawio/alphaomegarawio.py +++ b/neo/rawio/alphaomegarawio.py @@ -100,7 +100,7 @@ def __init__(self, dirname="", lsx_files=None, prune_channels=True): if self.dirname.is_dir(): self._explore_folder() else: - raise IOError(f"{self.dirname} is not a folder") + raise ValueError(f"{self.dirname} is not a folder") def _explore_folder(self): """ diff --git a/neo/test/rawiotest/test_alphaomegarawio.py b/neo/test/rawiotest/test_alphaomegarawio.py index 353f5435b..337418568 100644 --- a/neo/test/rawiotest/test_alphaomegarawio.py +++ b/neo/test/rawiotest/test_alphaomegarawio.py @@ -82,9 +82,8 @@ def test_explore_no_folder(self): with tempfile.TemporaryDirectory() as tmpdir: # just create a temporary folder that is removed pass - with self.assertLogs(logger=self.logger, level="ERROR") as cm: + with self.assertRaisesRegex(ValueError, "is not a folder"): reader = AlphaOmegaRawIO(dirname=tmpdir) - self.assertIn("is not a folder", cm.output[0]) def test_empty_folder(self): with tempfile.TemporaryDirectory() as tmpdir: From f83b586dee03aa225e49f785dd85efe54050e017 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Wed, 26 Oct 2022 08:43:07 +0200 Subject: [PATCH 09/26] [binarysignalio] don't list arbitrary extension to simplify `get_io` mechanism --- neo/rawio/rawbinarysignalrawio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/rawio/rawbinarysignalrawio.py b/neo/rawio/rawbinarysignalrawio.py index ef6b1c7a5..e71210df8 100644 --- a/neo/rawio/rawbinarysignalrawio.py +++ b/neo/rawio/rawbinarysignalrawio.py @@ -27,7 +27,7 @@ class RawBinarySignalIO class RawBinarySignalRawIO(BaseRawIO): - extensions = ['raw', '*'] + extensions = ['raw', 'bin'] rawmode = 'one-file' def __init__(self, filename='', dtype='int16', sampling_rate=10000., From 2cb076e00f4c98784350f0fe03fb1861a819d6e4 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Wed, 26 Oct 2022 14:04:57 +0200 Subject: [PATCH 10/26] Ensure fake example io test files exist to be able to run common io tests on them --- neo/test/iotest/test_exampleio.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/neo/test/iotest/test_exampleio.py b/neo/test/iotest/test_exampleio.py index 8b87cf585..5a0cf6602 100644 --- a/neo/test/iotest/test_exampleio.py +++ b/neo/test/iotest/test_exampleio.py @@ -2,10 +2,12 @@ Tests of neo.io.exampleio """ +import pathlib import unittest from neo.io.exampleio import ExampleIO # , HAVE_SCIPY from neo.test.iotest.common_io_test import BaseTestIO +from neo.test.iotest.tools import get_test_file_full_path from neo.io.proxyobjects import (AnalogSignalProxy, SpikeTrainProxy, EventProxy, EpochProxy) from neo import (AnalogSignal, SpikeTrain) @@ -19,10 +21,18 @@ class TestExampleIO(BaseTestIO, unittest.TestCase, ): ioclass = ExampleIO entities_to_download = [] entities_to_test = [ - 'fake1', - 'fake2', + 'fake1.fake', + 'fake2.fake', ] + def setUp(self): + super().setUp() + # ensure fake test files exist before running common tests + for entity in self.entities_to_test: + full_path = get_test_file_full_path(self.ioclass, filename=entity, + directory=self.local_test_dir) + pathlib.Path(full_path).touch() + # This is the minimal variables that are required # to run the common IO tests. IO specific tests # can be added here and will be run automatically From 1f4c22a95aed4414af57343bdcaa1410aeaee001 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Wed, 26 Oct 2022 14:28:25 +0200 Subject: [PATCH 11/26] clean up example io tests files after running common tests --- neo/test/iotest/test_exampleio.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/neo/test/iotest/test_exampleio.py b/neo/test/iotest/test_exampleio.py index 5a0cf6602..7dd3b2c76 100644 --- a/neo/test/iotest/test_exampleio.py +++ b/neo/test/iotest/test_exampleio.py @@ -33,6 +33,13 @@ def setUp(self): directory=self.local_test_dir) pathlib.Path(full_path).touch() + def tearDown(self) -> None: + super().tearDown() + for entity in self.entities_to_test: + full_path = get_test_file_full_path(self.ioclass, filename=entity, + directory=self.local_test_dir) + pathlib.Path(full_path).unlink(missing_ok=True) + # This is the minimal variables that are required # to run the common IO tests. IO specific tests # can be added here and will be run automatically From c2e17818c164b25d3e56cc6fc4cc7d3bfd87c729 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Wed, 26 Oct 2022 14:58:46 +0200 Subject: [PATCH 12/26] add NixIOFr to list of ios to be discoverable --- neo/io/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/neo/io/__init__.py b/neo/io/__init__.py index 599acfc8b..3b1573492 100644 --- a/neo/io/__init__.py +++ b/neo/io/__init__.py @@ -361,7 +361,8 @@ MEArecIO, MaxwellIO, MicromedIO, - NixIO, # place NixIO before other IOs that use HDF5 to make it the default for .h5 files + NixIO, + NixIOFr, NeoMatlabIO, NestIO, NeuralynxIO, From ccb6834691d35de01d48b3ed6ce81dbe31527621 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Wed, 26 Oct 2022 15:59:16 +0200 Subject: [PATCH 13/26] cleanup common_io_test unused code and typos --- neo/test/iotest/common_io_test.py | 72 +++++++------------------------ 1 file changed, 16 insertions(+), 56 deletions(-) diff --git a/neo/test/iotest/common_io_test.py b/neo/test/iotest/common_io_test.py index eea419bd6..eaaea9d35 100644 --- a/neo/test/iotest/common_io_test.py +++ b/neo/test/iotest/common_io_test.py @@ -19,7 +19,6 @@ import os import inspect -from copy import copy import unittest import pathlib @@ -27,13 +26,11 @@ from neo.io.basefromrawio import BaseFromRaw from neo.test.tools import (assert_same_sub_schema, assert_neo_object_is_compliant, - assert_sub_schema_is_lazy_loaded, - assert_children_empty) + assert_sub_schema_is_lazy_loaded) from neo.test.rawiotest.tools import can_use_network from neo.test.rawiotest.common_rawio_test import repo_for_test -from neo.utils import (download_dataset, - get_local_testing_data_folder) +from neo.utils import (download_dataset, get_local_testing_data_folder) from neo import list_candidate_ios try: @@ -42,10 +39,8 @@ except: HAVE_DATALAD = False -from neo.test.iotest.tools import (cleanup_test_file, - close_object_safe, create_generic_io_object, +from neo.test.iotest.tools import (close_object_safe, create_generic_io_object, create_generic_reader, - create_generic_writer, get_test_file_full_path, iter_generic_io_objects, iter_generic_readers, iter_read_objects, @@ -59,18 +54,17 @@ class BaseTestIO: ''' This class make common tests for all IOs. - Several startegies: + Several strategies: * for IO able to read write : test_write_then_read * for IO able to read write with hash conservation (optional): test_read_then_write - * for all IOs : test_assert_readed_neo_object_is_compliant + * for all IOs : test_assert_read_neo_object_is_compliant 2 cases: - * files are at G-node and downloaded: - download_test_files_if_not_present + * files are at G-node and downloaded * files are generated by MyIO.write() - ''' - # ~ __test__ = False + """ + __test__ = False # all IO test need to modify this: ioclass = None # the IOclass to be tested @@ -108,47 +102,12 @@ def setUp(self): self.files_generated = [] self.generate_files_for_io_able_to_write() - # be carefull self.entities_to_test is class attributes + # be careful self.entities_to_test is class attributes self.files_to_test = [self.get_local_path(e) for e in self.entities_to_test] else: self.files_to_test = [] raise unittest.SkipTest("Requires datalad download of data from the web") - def create_local_dir_if_not_exists(self): - ''' - Create a local directory to store testing files and return it. - - The directory path is also written to self.local_test_dir - ''' - self.local_test_dir = create_local_temp_dir( - self.shortname, directory=os.environ.get("NEO_TEST_FILE_DIR", None)) - return self.local_test_dir - - def download_test_files_if_not_present(self): - ''' - Download %s file at G-node for testing - url_for_tests is global at beginning of this file. - - ''' % self.ioclass.__name__ - if not self.use_network: - raise unittest.SkipTest("Requires download of data from the web") - - url = url_for_tests + self.shortname - try: - make_all_directories(self.files_to_download, self.local_test_dir) - download_test_file(self.files_to_download, - self.local_test_dir, url) - except OSError as exc: - raise unittest.TestCase.failureException(exc) - - download_test_files_if_not_present.__test__ = False - - def cleanup_file(self, path): - ''' - Remove test files or directories safely. - ''' - cleanup_test_file(self.ioclass, path, directory=self.local_test_dir) - def able_to_write_or_read(self, writeread=False, readwrite=False): ''' Return True if generalized writing or reading is possible. @@ -167,9 +126,9 @@ def able_to_write_or_read(self, writeread=False, readwrite=False): if self.higher not in [Block, Segment]: return False - # when io need external knowldge for writting or read such as - # sampling_rate (RawBinaryIO...) the test is too much complex to design - # genericaly. + # when io need external knowledge for writing or reading such as + # sampling_rate (RawBinaryIO...) the test is too complex to design + # generically. if (self.higher in self.ioclass.read_params and len(self.ioclass.read_params[self.higher]) != 0): return False @@ -184,7 +143,8 @@ def able_to_write_or_read(self, writeread=False, readwrite=False): return True - def get_local_base_folder(self): + @staticmethod + def get_local_base_folder(): return get_local_testing_data_folder() def get_local_path(self, sub_path): @@ -441,7 +401,7 @@ def test_read_then_write(self): return # assert_file_contents_equal(a, b) - def test_assert_readed_neo_object_is_compliant(self): + def test_assert_read_neo_object_is_compliant(self): ''' Reading %s files in `files_to_test` produces compliant objects. @@ -457,7 +417,7 @@ def test_assert_readed_neo_object_is_compliant(self): exc.args += ('from %s' % os.path.basename(path), ) raise - def test_readed_with_lazy_is_compliant(self): + def test_read_with_lazy_is_compliant(self): ''' Reading %s files in `files_to_test` with `lazy` is compliant. From 12d2eb40781f6e5b83c575de6c0d5a5d7213a3b2 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Wed, 26 Oct 2022 16:00:30 +0200 Subject: [PATCH 14/26] Test performance improvement: run test data fetching only once per TestClass and not once per test --- neo/test/iotest/common_io_test.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/neo/test/iotest/common_io_test.py b/neo/test/iotest/common_io_test.py index eaaea9d35..5df70a799 100644 --- a/neo/test/iotest/common_io_test.py +++ b/neo/test/iotest/common_io_test.py @@ -51,7 +51,7 @@ class BaseTestIO: - ''' + """ This class make common tests for all IOs. Several strategies: @@ -82,10 +82,8 @@ class BaseTestIO: local_test_dir = get_local_testing_data_folder() - def setUp(self): - ''' - Set up the test fixture. This is run for every test - ''' + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.higher = self.ioclass.supported_objects[0] self.shortname = self.ioclass.__name__.lower().rstrip('io') # these objects can both be written and read @@ -94,7 +92,6 @@ def setUp(self): # these objects can be either written or read self.io_readorwrite = list(set(self.ioclass.readable_objects) | set(self.ioclass.writeable_objects)) - if HAVE_DATALAD: for remote_path in self.entities_to_download: download_dataset(repo=repo_for_test, remote_path=remote_path) From c4378ad390660672eeb605992231ceb9eb3f7071 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Thu, 27 Oct 2022 12:26:01 +0200 Subject: [PATCH 15/26] [openephys] Add missing file extensions used --- neo/rawio/openephysbinaryrawio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/rawio/openephysbinaryrawio.py b/neo/rawio/openephysbinaryrawio.py index e59f89bdf..49df3499d 100644 --- a/neo/rawio/openephysbinaryrawio.py +++ b/neo/rawio/openephysbinaryrawio.py @@ -54,7 +54,7 @@ class OpenEphysBinaryRawIO(BaseRawIO): The current implementation does not handle spiking data, this will be added upon user request """ - extensions = [] + extensions = ['xml', 'oebin', 'txt', 'dat', 'npy'] rawmode = 'one-dir' def __init__(self, dirname='', load_sync_channel=False, experiment_names=None): From a1ba7280dad00be2613b956bfe6c2f5acecd8759 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Fri, 28 Oct 2022 17:00:22 +0200 Subject: [PATCH 16/26] Extend IO discovery to also check deeper subfolders to be compatible with nested-folder folders --- neo/io/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/neo/io/__init__.py b/neo/io/__init__.py index 3b1573492..96fd0411b 100644 --- a/neo/io/__init__.py +++ b/neo/io/__init__.py @@ -436,11 +436,19 @@ def list_candidate_ios(file_or_folder): # scan files in folder to determine io type filenames = [f for f in file_or_folder.glob('*') if f.is_file()] + # if no files are found in the folder, check subfolders + # this is necessary for nested-folder based formats like spikeglx + if not filenames: + filenames = [f for f in file_or_folder.glob('**/*') if f.is_file()] + # if only file prefix was provided, e.g /mydatafolder/session1- # to select all files sharing the `session1-` prefix elif file_or_folder.parent.exists(): filenames = file_or_folder.parent.glob(file_or_folder.name + '*') + else: + raise ValueError(f'{file_or_folder} does not contain data files of a supported format') + # find the io that fits the best with the files contained in the folder potential_ios = [] for filename in filenames: @@ -450,7 +458,7 @@ def list_candidate_ios(file_or_folder): potential_ios.extend(io_by_extension[suffix]) if not potential_ios: - raise ValueError(f'Could not determine io to load {file_or_folder}') + raise ValueError(f'Could not determine IO to load {file_or_folder}') # return ios ordered by number of files supported counter = Counter(potential_ios).most_common() From 4304fa863d6bb0bbafabc6adc38d13fbe80e5aff Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Wed, 2 Nov 2022 11:08:36 +0100 Subject: [PATCH 17/26] Undo BaseTestIO deactivation --- neo/test/iotest/common_io_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/test/iotest/common_io_test.py b/neo/test/iotest/common_io_test.py index 5df70a799..59e1bd3be 100644 --- a/neo/test/iotest/common_io_test.py +++ b/neo/test/iotest/common_io_test.py @@ -64,7 +64,7 @@ class BaseTestIO: * files are generated by MyIO.write() """ - __test__ = False + # __test__ = False # all IO test need to modify this: ioclass = None # the IOclass to be tested From 80610d1a2b2d168884da72fd9c67e29e74c122c4 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Wed, 2 Nov 2022 16:09:47 +0100 Subject: [PATCH 18/26] Fix typos --- neo/test/iotest/common_io_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/test/iotest/common_io_test.py b/neo/test/iotest/common_io_test.py index 59e1bd3be..0206c918a 100644 --- a/neo/test/iotest/common_io_test.py +++ b/neo/test/iotest/common_io_test.py @@ -9,7 +9,7 @@ The public URL is in url_for_tests. -To deposite new testing files, please create a account at +To deposit new testing files, please create a account at gin.g-node.org and upload files at NeuralEnsemble/ephy_testing_data data repo. @@ -52,7 +52,7 @@ class BaseTestIO: """ - This class make common tests for all IOs. + This class defines common tests for all IOs. Several strategies: * for IO able to read write : test_write_then_read From c5caf1e33733191f4d49b36feecd1bf44c1ec9bd Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Wed, 2 Nov 2022 16:52:42 +0100 Subject: [PATCH 19/26] Undo BaseTest restructuration --- neo/test/iotest/common_io_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/neo/test/iotest/common_io_test.py b/neo/test/iotest/common_io_test.py index 0206c918a..742a27a90 100644 --- a/neo/test/iotest/common_io_test.py +++ b/neo/test/iotest/common_io_test.py @@ -82,8 +82,10 @@ class BaseTestIO: local_test_dir = get_local_testing_data_folder() - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + def setUp(self): + ''' + Set up the test fixture. This is run for every test + ''' self.higher = self.ioclass.supported_objects[0] self.shortname = self.ioclass.__name__.lower().rstrip('io') # these objects can both be written and read From b82ebced721917fee688ae63827c8640e3a37dca Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Thu, 3 Nov 2022 11:05:35 +0100 Subject: [PATCH 20/26] introduce option to ignore system file extensions for more reliable io detection --- neo/io/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/neo/io/__init__.py b/neo/io/__init__.py index 96fd0411b..be845ffc9 100644 --- a/neo/io/__init__.py +++ b/neo/io/__init__.py @@ -410,7 +410,7 @@ def get_io(file_or_folder, *args, **kwargs): raise IOError(f"Could not identify IO for {file_or_folder}") -def list_candidate_ios(file_or_folder): +def list_candidate_ios(file_or_folder, ignore_suffix=['ini']): """ Identify neo IO that can potentially load data in the file or folder @@ -418,6 +418,8 @@ def list_candidate_ios(file_or_folder): ---------- file_or_folder (str, pathlib.Path) Path to the file or folder to load + ignore_suffix (list) + List of suffixes to ignore when scanning for known formats. Default: ['ini'] Returns ------- @@ -435,11 +437,15 @@ def list_candidate_ios(file_or_folder): elif file_or_folder.is_dir(): # scan files in folder to determine io type filenames = [f for f in file_or_folder.glob('*') if f.is_file()] + # keep only relevant filenames + filenames = [f for f in filenames if f.suffix and f.suffix[1:].lower() not in ignore_suffix] # if no files are found in the folder, check subfolders # this is necessary for nested-folder based formats like spikeglx if not filenames: filenames = [f for f in file_or_folder.glob('**/*') if f.is_file()] + # keep only relevant filenames + filenames = [f for f in filenames if f.suffix and f.suffix[1:].lower() not in ignore_suffix] # if only file prefix was provided, e.g /mydatafolder/session1- # to select all files sharing the `session1-` prefix From be8e92aa040a17244b05a07d5bc0eb6fe55edc9f Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Thu, 3 Nov 2022 11:13:14 +0100 Subject: [PATCH 21/26] fix outdated test file names for spikeglx --- neo/test/rawiotest/test_spikeglxrawio.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/neo/test/rawiotest/test_spikeglxrawio.py b/neo/test/rawiotest/test_spikeglxrawio.py index 62a2f13b2..e4fd3a810 100644 --- a/neo/test/rawiotest/test_spikeglxrawio.py +++ b/neo/test/rawiotest/test_spikeglxrawio.py @@ -18,17 +18,17 @@ class TestSpikeGLXRawIO(BaseTestRawIO, unittest.TestCase): 'spikeglx/TEST_20210920_0_g0', # this is only g0 multi index - 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0/5-19-2022-CI0_g0' + 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0/5-19-2022-CI0_g0', # this is only g1 multi index - 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0/5-19-2022-CI0_g1' + 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0/5-19-2022-CI0_g1', # this mix both multi gate and multi trigger (and also multi probe) - 'spikeglx/sample_data_v2/SpikeGLX/5-19-2022-CI0', + 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI0', - 'spikeglx/sample_data_v2/SpikeGLX/5-19-2022-CI1', - 'spikeglx/sample_data_v2/SpikeGLX/5-19-2022-CI2', - 'spikeglx/sample_data_v2/SpikeGLX/5-19-2022-CI3', - 'spikeglx/sample_data_v2/SpikeGLX/5-19-2022-CI4', - 'spikeglx/sample_data_v2/SpikeGLX/5-19-2022-CI5', + 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI1', + 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI2', + 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI3', + 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI4', + 'spikeglx/multi_trigger_multi_gate/SpikeGLX/5-19-2022-CI5', ] From 8c7ff7759bd1742706256fcb6419d74902f008e0 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Thu, 3 Nov 2022 11:31:54 +0100 Subject: [PATCH 22/26] raise error if format is not linked to any IO --- neo/io/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/io/__init__.py b/neo/io/__init__.py index be845ffc9..5cb73bea7 100644 --- a/neo/io/__init__.py +++ b/neo/io/__init__.py @@ -431,7 +431,7 @@ def list_candidate_ios(file_or_folder, ignore_suffix=['ini']): if file_or_folder.is_file(): suffix = file_or_folder.suffix[1:].lower() if suffix not in io_by_extension: - print(f'{suffix} not found') + raise ValueError(f'{suffix} is not a supported format of any IO.') return io_by_extension[suffix] elif file_or_folder.is_dir(): From faf073e99ea37597e09ace35132de2415e492847 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Thu, 3 Nov 2022 12:42:17 +0100 Subject: [PATCH 23/26] add missing extension for neurosharectypesio --- neo/io/neurosharectypesio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/io/neurosharectypesio.py b/neo/io/neurosharectypesio.py index 4645316e5..1baf8fd0f 100644 --- a/neo/io/neurosharectypesio.py +++ b/neo/io/neurosharectypesio.py @@ -108,7 +108,7 @@ class NeurosharectypesIO(BaseIO): write_params = None name = 'neuroshare' - extensions = [] + extensions = ['mcd'] mode = 'file' def __init__(self, filename='', dllname=''): From 84d10d7991e8f8bd8f3aeed1ceb1382618b96629 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Thu, 3 Nov 2022 15:37:31 +0100 Subject: [PATCH 24/26] use ignore pattern instead of ignore suffix for more flexibility (e.g. for README files) --- neo/io/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/neo/io/__init__.py b/neo/io/__init__.py index 5cb73bea7..22b6df662 100644 --- a/neo/io/__init__.py +++ b/neo/io/__init__.py @@ -410,7 +410,7 @@ def get_io(file_or_folder, *args, **kwargs): raise IOError(f"Could not identify IO for {file_or_folder}") -def list_candidate_ios(file_or_folder, ignore_suffix=['ini']): +def list_candidate_ios(file_or_folder, ignore_patterns=['*.ini', 'README.txt', 'README.md']): """ Identify neo IO that can potentially load data in the file or folder @@ -418,8 +418,9 @@ def list_candidate_ios(file_or_folder, ignore_suffix=['ini']): ---------- file_or_folder (str, pathlib.Path) Path to the file or folder to load - ignore_suffix (list) - List of suffixes to ignore when scanning for known formats. Default: ['ini'] + ignore_patterns (list) + List of patterns to ignore when scanning for known formats. See pathlib.PurePath.match(). + Default: ['ini'] Returns ------- @@ -438,14 +439,14 @@ def list_candidate_ios(file_or_folder, ignore_suffix=['ini']): # scan files in folder to determine io type filenames = [f for f in file_or_folder.glob('*') if f.is_file()] # keep only relevant filenames - filenames = [f for f in filenames if f.suffix and f.suffix[1:].lower() not in ignore_suffix] + filenames = [f for f in filenames if f.suffix and not any([f.match(p) for p in ignore_patterns])] # if no files are found in the folder, check subfolders # this is necessary for nested-folder based formats like spikeglx if not filenames: filenames = [f for f in file_or_folder.glob('**/*') if f.is_file()] # keep only relevant filenames - filenames = [f for f in filenames if f.suffix and f.suffix[1:].lower() not in ignore_suffix] + filenames = [f for f in filenames if f.suffix and not any([f.match(p) for p in ignore_patterns])] # if only file prefix was provided, e.g /mydatafolder/session1- # to select all files sharing the `session1-` prefix From 0da43f7bc3c544a0a21442dd763a3bead7f61288 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Thu, 3 Nov 2022 15:37:49 +0100 Subject: [PATCH 25/26] remove print statement --- neo/test/iotest/common_io_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/neo/test/iotest/common_io_test.py b/neo/test/iotest/common_io_test.py index 742a27a90..b7266532b 100644 --- a/neo/test/iotest/common_io_test.py +++ b/neo/test/iotest/common_io_test.py @@ -494,6 +494,4 @@ def test_list_candidate_ios(self): entity = get_test_file_full_path(self.ioclass, filename=entity, directory=self.local_test_dir) ios = list_candidate_ios(entity) - if self.ioclass not in ios: - print(f'{entity} does not list {self.ioclass} in list of ios: {ios}') assert self.ioclass in ios From fdb2fca0ce0bee3dbcb5b467a478bfc5bd99ac25 Mon Sep 17 00:00:00 2001 From: Julia Sprenger Date: Mon, 19 Dec 2022 15:11:11 +0100 Subject: [PATCH 26/26] Update neo/test/iotest/common_io_test.py --- neo/test/iotest/common_io_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/test/iotest/common_io_test.py b/neo/test/iotest/common_io_test.py index 81845abd3..bbe0986da 100644 --- a/neo/test/iotest/common_io_test.py +++ b/neo/test/iotest/common_io_test.py @@ -514,7 +514,7 @@ def test__handle_pathlib_filename(self): elif self.ioclass.mode == 'dir': self.ioclass(dirname=pathlib_filename, *self.default_arguments, - **self.default_keyword_arguments)) + **self.default_keyword_arguments) def test_list_candidate_ios(self): for entity in self.entities_to_test: