Merge pull request #31 from isi-nmr/dev_JV

Optimizations of jcampdx readin, faster file transfer, Support for rawdata, fix EPSI and dtiEPI fids bump version

Author: headmeister

README.rst

@@ -164,54 +164,7 @@ Compatible pulse sequences for **fid** data sets:
 * STEAM.ppg
 * igFLASH.ppg
 
-ParaVision v6.0.1
-"""""""""""""""""
-Compatible data set types:
-
-* **fid**
-* **2dseq**
-* **rawdata.job0**
-* **rawdata.Navigator**
-
-Compatible pulse sequences for **fid** data sets:
-
-* FLASH.ppg,
-* FLASHAngio.ppg
-* IgFLASH.ppg
-* MGE.ppg
-* MSME.ppg
-* RARE.ppg
-* FAIR_RARE.ppg
-* RAREVTR.ppg
-* RAREst.ppg
-* MDEFT.ppg
-* FISP.ppg
-* FLOWMAP.ppg
-* DtiStandard.ppg
-* EPI.ppg
-* FAIR_EPI.ppg
-* CASL_EPI.ppg
-* DtiEpi.ppg
-* T1_EPI.ppg
-* T2_EPI.ppg
-* T2S_EPI.ppg
-* SPIRAL.ppg
-* DtiSpiral.ppg
-* UTE.ppg
-* UTE3D.ppg
-* ZTE.ppg
-* CSI.ppg
-* FieldMap.ppg
-* SINGLEPULSE.ppg
-* NSPECT.ppg
-* EPSI.ppg
-* PRESS.ppg
-* STEAM.ppg
-* ISIS.ppg
-* CPMG.ppg
-* RfProfile.ppg
-
-ParaVision v7.0.0
+ParaVision v6.0.1 and v7.0.0
 """""""""""""""""
 Compatible data set types:
 
@@ -259,8 +212,9 @@ Compatible pulse sequences for **fid** data sets:
 * RfProfile.ppg
 
 
-ParaVision v360
+ParaVision 360 v1.1 v3.0-v3.7
 """""""""""""""
+Reading rawdata is supported only in a basic form, no reshaping into k-space is supported at the moment.
 Compatible data set types:
 
 * **2dseq**

brukerapi/config/properties_fid_core.json

@@ -223,9 +223,7 @@
         ["#PULPROG[1:-1]",
           [
             "EPI.ppg",
-            "DtiEpi.ppg",
             "navigatorEPI_OM.ppg",
-            "EPSI.ppg",
             "FAIR_EPI.ppg",
             "CASL_EPI.ppg",
             "T1_EPI.ppg",
@@ -236,6 +234,18 @@
         "#ACQ_sw_version in ['<PV 5.1>', '<PV 6.0>', '<PV 6.0.1>', '<PV-7.0.0>']"
       ]
     },
+    {
+      "cmd": "'dEPI'" ,
+      "conditions": [
+        ["#PULPROG[1:-1]",
+          [
+            "DtiEpi.ppg",
+            "EPSI.ppg"
+          ]
+        ],
+        "#ACQ_sw_version in ['<PV 5.1>', '<PV 6.0>', '<PV 6.0.1>', '<PV-7.0.0>']"
+      ]
+    },
     {
       "cmd": "'SPECTROSCOPY'" ,
       "conditions": [
@@ -322,7 +332,13 @@
       ]
     },
     {
-      "cmd": "#NR",
+      "cmd": "#NSegments*#NI*#NR*(#ACQ_size[2] if len(#ACQ_size)>2 else 1)",
+      "conditions": [
+        "@scheme_id=='dEPI'"
+      ]
+    },
+    {
+      "cmd": "#NR*#NI",
       "conditions": [
         "@scheme_id=='SPECTROSCOPY'"
       ]
@@ -430,9 +446,23 @@
         "@scheme_id=='EPI'"
       ]
     },
+    {
+      "cmd": [
+        "#PVM_EncMatrix[0] * #PVM_EncMatrix[1] // #NSegments",
+        "#PVM_EncNReceivers",
+        "#NSegments",
+        "#NI",
+        "#NR",
+        "#ACQ_size[2] if len(#ACQ_size)>2 else 1"
+      ],
+      "conditions": [
+        "@scheme_id=='dEPI'"
+      ]
+    },
     {
       "cmd": [
         "#ACQ_size.tuple[0] // 2",
+        "#NI",
         "#NR"
       ],
       "conditions": [
@@ -527,11 +557,24 @@
     {
       "cmd": [0,2,3,4,1],
       "conditions": [
-        "@scheme_id in ['EPI', 'SPIRAL']"
+        "@scheme_id in ['SPIRAL']"
+      ]
+    },
+    {
+      "cmd": [0,2,3,4,1],
+      "conditions": [
+        "@scheme_id in ['EPI']"
+      ]
+    },
+    {
+      "cmd": [0,2,3,4,1,5],
+      "conditions": [
+        "@scheme_id in ['dEPI']"
       ]
     },
+    
     {
-      "cmd": [0,1],
+      "cmd": [0,1,2],
       "conditions": [
         "@scheme_id=='SPECTROSCOPY'"
       ]
@@ -604,9 +647,23 @@
         "@scheme_id=='EPI'"
       ]
     },
+    {
+      "cmd": [
+        "#PVM_EncMatrix[0]",
+        "#PVM_EncMatrix[1]",
+        "#NI",
+        "#NR",
+        "#PVM_EncNReceivers",
+        "#ACQ_size[2] if len(#ACQ_size)>2 else 1"
+      ],
+      "conditions": [
+        "@scheme_id=='dEPI'"
+      ]
+    },
     {
       "cmd": [
         "#ACQ_size.tuple[0] // 2",
+        "#NI",
         "#NR"
       ],
       "conditions": [
@@ -701,7 +758,7 @@
         "'channel'"
       ],
       "conditions": [
-        ["@scheme_id",["CART_2D","RADIAL","EPI","SPIRAL","ZTE"]]
+        ["@scheme_id",["CART_2D","RADIAL","EPI","dEPI","SPIRAL","ZTE"]]
       ]
     },
     {

brukerapi/config/properties_rawdata_core.json

@@ -5,65 +5,72 @@
         "conditions": [
           "#GO_raw_data_format=='GO_32BIT_SGN_INT'",
           "#BYTORDA=='little'",
-          ["#ACQ_sw_version",["<PV 5.1>", "<PV 6.0>", "<PV 6.0.1>"]]
+          ["#ACQ_sw_version",["<PV 5.1>", "<PV 6.0>", "<PV 6.0.1>","<PV-7.0.0>"]]
         ]
     },
     {
         "cmd": "np.dtype('int16').newbyteorder('<')",
         "conditions": [
           "#GO_raw_data_format=='GO_16BIT_SGN_INT'",
           "#BYTORDA=='little'",
-          ["#ACQ_sw_version",["<PV 5.1>", "<PV 6.0>", "<PV 6.0.1>"]]
+          ["#ACQ_sw_version",["<PV 5.1>", "<PV 6.0>", "<PV 6.0.1>","<PV-7.0.0>"]]
         ]
     },
     {
         "cmd": "np.dtype('float32').newbyteorder('<')",
         "conditions": [
           "#GO_raw_data_format=='GO_32BIT_FLOAT'",
           "#BYTORDA=='little'",
-          ["#ACQ_sw_version",["<PV 5.1>", "<PV 6.0>", "<PV 6.0.1>"]]
+          ["#ACQ_sw_version",["<PV 5.1>", "<PV 6.0>", "<PV 6.0.1>","<PV-7.0.0>"]]
         ]
     },
     {
         "cmd": "np.dtype('int32').newbyteorder('>')",
         "conditions": [
           "#GO_raw_data_format=='GO_32BIT_SGN_INT'",
           "#BYTORDA=='big'",
-          ["#ACQ_sw_version",["<PV 5.1>", "<PV 6.0>", "<PV 6.0.1>"]]
+          ["#ACQ_sw_version",["<PV 5.1>", "<PV 6.0>", "<PV 6.0.1>","<PV-7.0.0>"]]
         ]
     },
     {
         "cmd": "np.dtype('int16').newbyteorder('>')",
         "conditions": [
           "#GO_raw_data_format=='GO_16BIT_SGN_INT'",
           "#BYTORDA=='big'",
-          ["#ACQ_sw_version",["<PV 5.1>", "<PV 6.0>", "<PV 6.0.1>"]]
+          ["#ACQ_sw_version",["<PV 5.1>", "<PV 6.0>", "<PV 6.0.1>","<PV-7.0.0>"]]
         ]
     },
     {
         "cmd": "np.dtype('float32').newbyteorder('>')",
         "conditions": [
           "#GO_raw_data_format=='GO_32BIT_FLOAT'",
           "#BYTORDA=='big'",
-          ["#ACQ_sw_version",["<PV 5.1>", "<PV 6.0>", "<PV 6.0.1>"]]
+          ["#ACQ_sw_version",["<PV 5.1>", "<PV 6.0>", "<PV 6.0.1>","<PV-7.0.0>"]]
         ]
     },
     {
         "cmd": "np.dtype('i4').newbyteorder('<')",
         "conditions": [
           "#ACQ_word_size=='_32_BIT'",
           "#BYTORDA=='little'",
-          ["#ACQ_sw_version",["<PV-360.1.1>"]]
+          ["#ACQ_sw_version",["<PV-360.1.1>","<PV-360.3.0>","<PV-360.3.1>","<PV-360.3.2>","<PV-360.3.3>","<PV-360.3.4>","<PV-360.3.5>","<PV-360.3.6>","<PV-360.3.7>"]]
         ]
     }
   ],
   "job_desc": [
     {
       "cmd": "#ACQ_jobs.primed_dict(7)['<{}>'.format(@subtype)]",
       "conditions": [
-        "#ACQ_sw_version in ['<PV-360.1.1>']"
+        ["#ACQ_sw_version",["<PV-360.1.1>"]]
       ]
     },
+    {
+      "cmd": "[v for v in #ACQ_jobs.nested if v[-1] == '<{}>'.format(@subtype)][0]",
+      "conditions": [
+        ["#ACQ_sw_version",["<PV-360.3.0>","<PV-360.3.1>","<PV-360.3.2>","<PV-360.3.3>","<PV-360.3.4>","<PV-360.3.5>","<PV-360.3.6>","<PV-360.3.7>"]]
+      ]
+    },
+
     {
       "cmd": "#ACQ_jobs.nested[0]",
       "conditions": [
@@ -88,7 +95,7 @@
   ],
   "shape_storage": [
     {
-      "cmd": "(@job_desc[0],) + (@job_desc[3],)",
+      "cmd": "(@job_desc[0],) + (#PVM_EncNReceivers,) + (@job_desc[3],)",
       "conditions": []
     }
   ]

brukerapi/config/properties_rawdata_custom.json

@@ -1 +1,65 @@
-{}
+{
+  "subj_id": [
+    {
+      "cmd": "#SUBJECT_id[1:-1]",
+      "conditions": [
+
+      ]
+    },
+    {
+      "cmd": "''",
+      "conditions": [
+
+      ]
+    }
+  ],
+  "study_id": [
+    {
+      "cmd": "str(#SUBJECT_study_nr)",
+      "conditions": [
+
+      ]
+    },
+    {
+      "cmd": "''",
+      "conditions": [
+
+      ]
+    }
+  ],
+  "exp_id": [
+    {
+      "cmd": "@path.parent.name",
+      "conditions": [
+
+      ]
+    },
+    {
+      "cmd": "''",
+      "conditions": [
+
+      ]
+    }
+  ],
+  "id": [
+    {
+      "cmd": "f'RawData_{@subtype}_{@exp_id}_{@subj_id}_{@study_id}'",
+      "conditions": [
+      ]
+    }
+  ],
+  "TR": [
+    {
+        "cmd": "#PVM_RepetitionTime",
+        "conditions": [],
+        "unit": "ms"
+    }
+  ],
+  "TE": [
+    {
+        "cmd": "#PVM_EchoTime",
+        "conditions": [],
+        "unit": "ms"
+    }
+  ]
+}

brukerapi/dataset.py

@@ -546,10 +546,10 @@ def _read_binary_file(self, path, dtype, shape):
         1D ndarray containing the full data vector
         """
         # TODO debug with this
-        # try:
-        #     assert os.stat(str(path)).st_size == np.prod(shape) * dtype.itemsize
-        # except AssertionError:
-        #     raise ValueError('Dimension missmatch')
+        try:
+            assert os.stat(str(path)).st_size == np.prod(shape) * dtype.itemsize
+        except AssertionError:
+            raise ValueError("Dimension mismatch") from AssertionError
 
         return np.array(np.memmap(path, dtype=dtype, shape=shape, order="F")[:])
 
@@ -594,7 +594,7 @@ def write(self, path, **kwargs):
 
         path = Path(path)
 
-        if path.name != self.type:
+        if path.name.split(".")[0] != self.type:
             raise DatasetTypeMissmatch
 
         parent = path.parent

brukerapi/folders.py

@@ -217,7 +217,7 @@ def make_tree(self, *, recursive: bool = True) -> list:
                 children.append(Folder(path, parent=self, recursive=recursive, dataset_index=self._dataset_index, dataset_state=self._dataset_state))
                 continue
 
-            if path.name in self._dataset_index:
+            if path.name in self._dataset_index or (path.name.partition(".")[0] in self._dataset_index and "rawdata" in path.name):
                 try:
                     children.append(Dataset(path, **self._dataset_state))
                 except (UnsuportedDatasetType, IncompleteDataset, NotADatasetDir):