diff --git a/examples/data/dolfyn/test_data/BenchFile01.nc b/examples/data/dolfyn/test_data/BenchFile01.nc index 3b2af8fc..b12e6318 100644 Binary files a/examples/data/dolfyn/test_data/BenchFile01.nc and b/examples/data/dolfyn/test_data/BenchFile01.nc differ diff --git a/examples/data/dolfyn/test_data/BenchFile01.repr.txt b/examples/data/dolfyn/test_data/BenchFile01.repr.txt index 5e5d563c..47e5983f 100644 --- a/examples/data/dolfyn/test_data/BenchFile01.repr.txt +++ b/examples/data/dolfyn/test_data/BenchFile01.repr.txt @@ -17,10 +17,9 @@ - pressure ('time',) - pressure_alt ('time',) - pressure_altraw ('time_altraw',) + - pressure_b5 ('time_b5',) - amp ('beam', 'range', 'time') - amp_b5 ('range_b5', 'time_b5') - corr ('beam', 'range', 'time') - corr_b5 ('range_b5', 'time_b5') - - accel ('dirIMU', 'time') - - mag ('dirIMU', 'time') ... and others (see `.variables`) diff --git a/examples/data/dolfyn/test_data/BenchFile01_avg.nc b/examples/data/dolfyn/test_data/BenchFile01_avg.nc index b00515e7..a0d22d0e 100644 Binary files a/examples/data/dolfyn/test_data/BenchFile01_avg.nc and b/examples/data/dolfyn/test_data/BenchFile01_avg.nc differ diff --git a/examples/data/dolfyn/test_data/BenchFile01_crop.nc b/examples/data/dolfyn/test_data/BenchFile01_crop.nc index 278a872b..552aedad 100644 Binary files a/examples/data/dolfyn/test_data/BenchFile01_crop.nc and b/examples/data/dolfyn/test_data/BenchFile01_crop.nc differ diff --git a/examples/data/dolfyn/test_data/BenchFile01_rotate_beam2inst.nc b/examples/data/dolfyn/test_data/BenchFile01_rotate_beam2inst.nc index 2004de5f..6c6339fd 100644 Binary files a/examples/data/dolfyn/test_data/BenchFile01_rotate_beam2inst.nc and b/examples/data/dolfyn/test_data/BenchFile01_rotate_beam2inst.nc differ diff --git a/examples/data/dolfyn/test_data/BenchFile01_rotate_earth2principal.nc b/examples/data/dolfyn/test_data/BenchFile01_rotate_earth2principal.nc index a71cbbbd..8612e7bf 100644 Binary files a/examples/data/dolfyn/test_data/BenchFile01_rotate_earth2principal.nc and b/examples/data/dolfyn/test_data/BenchFile01_rotate_earth2principal.nc differ diff --git a/examples/data/dolfyn/test_data/BenchFile01_rotate_inst2earth.nc b/examples/data/dolfyn/test_data/BenchFile01_rotate_inst2earth.nc index bbfeaf37..c26e55d5 100644 Binary files a/examples/data/dolfyn/test_data/BenchFile01_rotate_inst2earth.nc and b/examples/data/dolfyn/test_data/BenchFile01_rotate_inst2earth.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_BadTime01.nc b/examples/data/dolfyn/test_data/Sig1000_BadTime01.nc index de929e2e..f4a30903 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_BadTime01.nc and b/examples/data/dolfyn/test_data/Sig1000_BadTime01.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_IMU.nc b/examples/data/dolfyn/test_data/Sig1000_IMU.nc index 397475f5..ee53b2ad 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_IMU.nc and b/examples/data/dolfyn/test_data/Sig1000_IMU.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_IMU_ofilt.nc b/examples/data/dolfyn/test_data/Sig1000_IMU_ofilt.nc index b46762be..be3fd624 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_IMU_ofilt.nc and b/examples/data/dolfyn/test_data/Sig1000_IMU_ofilt.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_beam2inst.nc b/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_beam2inst.nc index 6c333715..0f41b34f 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_beam2inst.nc and b/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_beam2inst.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_inst2earth.nc b/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_inst2earth.nc index f7364c5f..48e63e43 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_inst2earth.nc and b/examples/data/dolfyn/test_data/Sig1000_IMU_rotate_inst2earth.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_IMU_ud.nc b/examples/data/dolfyn/test_data/Sig1000_IMU_ud.nc index 05663f60..e22599a0 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_IMU_ud.nc and b/examples/data/dolfyn/test_data/Sig1000_IMU_ud.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_dp_echo1.nc b/examples/data/dolfyn/test_data/Sig1000_dp_echo1.nc index 28cef003..4f01f839 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_dp_echo1.nc and b/examples/data/dolfyn/test_data/Sig1000_dp_echo1.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_dp_echo2.nc b/examples/data/dolfyn/test_data/Sig1000_dp_echo2.nc index d2fb9f4e..51cabb1a 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_dp_echo2.nc and b/examples/data/dolfyn/test_data/Sig1000_dp_echo2.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_tidal.nc b/examples/data/dolfyn/test_data/Sig1000_tidal.nc index 759fe477..e2f2b2c1 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_tidal.nc and b/examples/data/dolfyn/test_data/Sig1000_tidal.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_tidal_bin.nc b/examples/data/dolfyn/test_data/Sig1000_tidal_bin.nc index b7a157d3..7b76e589 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_tidal_bin.nc and b/examples/data/dolfyn/test_data/Sig1000_tidal_bin.nc differ diff --git a/examples/data/dolfyn/test_data/Sig1000_tidal_clean.nc b/examples/data/dolfyn/test_data/Sig1000_tidal_clean.nc index b78e028c..d6bad8a9 100644 Binary files a/examples/data/dolfyn/test_data/Sig1000_tidal_clean.nc and b/examples/data/dolfyn/test_data/Sig1000_tidal_clean.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_Echo.dolfyn.log b/examples/data/dolfyn/test_data/Sig500_Echo.dolfyn.log index 2bfe04ac..ef60a78f 100644 --- a/examples/data/dolfyn/test_data/Sig500_Echo.dolfyn.log +++ b/examples/data/dolfyn/test_data/Sig500_Echo.dolfyn.log @@ -1,30 +1,101 @@ root - INFO - 4464: A5, 10, 18, 188, 0, 1, 1 - root - INFO - 4662: A5, 10, 15, 524, 0, 1, 1 - root - INFO - 5196: A5, 10, 1C, 214, 0, 1, 1 - -root - INFO - 5420: A5, 10, 18, 188, 1, 2, 2 - -root - INFO - 5618: A5, 10, 15, 524, 1, 2, 2 - -root - INFO - 6152: A5, 10, 1C, 214, 1, 2, 2 - -root - INFO - 6376: A5, 10, 18, 188, 2, 3, 3 - -root - INFO - 6574: A5, 10, 15, 524, 2, 3, 3 - -root - INFO - 7108: A5, 10, 1C, 214, 2, 3, 3 - -root - INFO - 7332: A5, 10, 18, 188, 3, 4, 4 - -root - INFO - 7530: A5, 10, 15, 524, 3, 4, 4 - -root - INFO - 8064: A5, 10, 1C, 214, 3, 4, 4 - -root - INFO - 8288: A5, 10, 18, 188, 4, 5, 5 - -root - INFO - 8486: A5, 10, 15, 524, 4, 5, 5 - -root - INFO - 9020: A5, 10, 1C, 214, 4, 5, 5 - +root - INFO - 5420: A5, 10, 18, 188, 1, 1, 1 +root - INFO - 5618: A5, 10, 15, 524, 1, 1, 1 +root - INFO - 6152: A5, 10, 1C, 214, 1, 1, 1 +root - INFO - 6376: A5, 10, 18, 188, 2, 1, 1 +root - INFO - 6574: A5, 10, 15, 524, 2, 1, 1 +root - INFO - 7108: A5, 10, 1C, 214, 2, 1, 1 +root - INFO - 7332: A5, 10, 18, 188, 3, 1, 1 +root - INFO - 7530: A5, 10, 15, 524, 3, 1, 1 +root - INFO - 8064: A5, 10, 1C, 214, 3, 1, 1 +root - INFO - 8288: A5, 10, 18, 188, 4, 1, 1 +root - INFO - 8486: A5, 10, 15, 524, 4, 1, 1 +root - INFO - 9020: A5, 10, 1C, 214, 4, 1, 1 +root - INFO - 9244: A5, 10, 18, 188, 5, 1, 1 +root - INFO - 9442: A5, 10, 15, 524, 5, 1, 1 +root - INFO - 9976: A5, 10, 1C, 214, 5, 1, 1 +root - INFO - 10200: A5, 10, 18, 188, 6, 1, 1 +root - INFO - 10398: A5, 10, 15, 524, 6, 1, 1 +root - INFO - 10932: A5, 10, 1C, 214, 6, 1, 1 +root - INFO - 11156: A5, 10, 18, 188, 7, 1, 1 +root - INFO - 11354: A5, 10, 15, 524, 7, 1, 1 +root - INFO - 11888: A5, 10, 1C, 214, 7, 1, 1 +root - INFO - 12112: A5, 10, 18, 188, 8, 1, 1 +root - INFO - 12310: A5, 10, 15, 524, 8, 1, 1 +root - INFO - 12844: A5, 10, 1C, 214, 8, 1, 1 +root - INFO - 13068: A5, 10, 18, 188, 9, 1, 1 +root - INFO - 13266: A5, 10, 15, 524, 9, 1, 1 +root - INFO - 13800: A5, 10, 1C, 214, 9, 1, 1 +root - INFO - 14024: A5, 10, 18, 188, 10, 1, 1 +root - INFO - 14222: A5, 10, 15, 524, 10, 1, 1 +root - INFO - 14756: A5, 10, 1C, 214, 10, 1, 1 +root - INFO - 14980: A5, 10, 18, 188, 11, 1, 1 +root - INFO - 15178: A5, 10, 15, 524, 11, 1, 1 +root - INFO - 15712: A5, 10, 1C, 214, 11, 1, 1 +root - INFO - 15936: A5, 10, 18, 188, 12, 1, 1 +root - INFO - 16134: A5, 10, 15, 524, 12, 1, 1 +root - INFO - 16668: A5, 10, 1C, 214, 12, 1, 1 +root - INFO - 16892: A5, 10, 18, 188, 13, 1, 1 +root - INFO - 17090: A5, 10, 15, 524, 13, 1, 1 +root - INFO - 17624: A5, 10, 1C, 214, 13, 1, 1 +root - INFO - 17848: A5, 10, 18, 188, 14, 1, 1 +root - INFO - 18046: A5, 10, 15, 524, 14, 1, 1 +root - INFO - 18580: A5, 10, 1C, 214, 14, 1, 1 +root - INFO - 18804: A5, 10, 18, 188, 15, 1, 1 +root - INFO - 19002: A5, 10, 15, 524, 15, 1, 1 +root - INFO - 19536: A5, 10, 1C, 214, 15, 1, 1 +root - INFO - 19760: A5, 10, 18, 188, 16, 1, 1 +root - INFO - 19958: A5, 10, 15, 524, 16, 1, 1 +root - INFO - 20492: A5, 10, 1C, 214, 16, 1, 1 +root - INFO - 20716: A5, 10, 18, 188, 17, 1, 1 +root - INFO - 20914: A5, 10, 15, 524, 17, 1, 1 +root - INFO - 21448: A5, 10, 1C, 214, 17, 1, 1 +root - INFO - 21672: A5, 10, 18, 188, 18, 1, 1 +root - INFO - 21870: A5, 10, 15, 524, 18, 1, 1 +root - INFO - 22404: A5, 10, 1C, 214, 18, 1, 1 +root - INFO - 22628: A5, 10, 18, 188, 19, 1, 1 +root - INFO - 22826: A5, 10, 15, 524, 19, 1, 1 +root - INFO - 23360: A5, 10, 1C, 214, 19, 1, 1 +root - INFO - 23584: A5, 10, 18, 188, 20, 1, 1 +root - INFO - 23782: A5, 10, 15, 524, 20, 1, 1 +root - INFO - 24316: A5, 10, 1C, 214, 20, 1, 1 +root - INFO - 24540: A5, 10, 18, 188, 21, 1, 1 +root - INFO - 24738: A5, 10, 15, 524, 21, 1, 1 +root - INFO - 25272: A5, 10, 1C, 214, 21, 1, 1 +root - INFO - 25496: A5, 10, 18, 188, 22, 1, 1 +root - INFO - 25694: A5, 10, 15, 524, 22, 1, 1 +root - INFO - 26228: A5, 10, 1C, 214, 22, 1, 1 +root - INFO - 26452: A5, 10, 18, 188, 23, 1, 1 +root - INFO - 26650: A5, 10, 15, 524, 23, 1, 1 +root - INFO - 27184: A5, 10, 1C, 214, 23, 1, 1 +root - INFO - 27408: A5, 10, 18, 188, 24, 1, 1 +root - INFO - 27606: A5, 10, 15, 524, 24, 1, 1 +root - INFO - 28140: A5, 10, 1C, 214, 24, 1, 1 +root - INFO - 28364: A5, 10, 18, 188, 25, 1, 1 +root - INFO - 28562: A5, 10, 15, 524, 25, 1, 1 +root - INFO - 29096: A5, 10, 1C, 214, 25, 1, 1 +root - INFO - 29320: A5, 10, 18, 188, 26, 1, 1 +root - INFO - 29518: A5, 10, 15, 524, 26, 1, 1 +root - INFO - 30052: A5, 10, 1C, 214, 26, 1, 1 +root - INFO - 30276: A5, 10, 18, 188, 27, 1, 1 +root - INFO - 30474: A5, 10, 15, 524, 27, 1, 1 +root - INFO - 31008: A5, 10, 1C, 214, 27, 1, 1 +root - INFO - 31232: A5, 10, 18, 188, 28, 1, 1 +root - INFO - 31430: A5, 10, 15, 524, 28, 1, 1 +root - INFO - 31964: A5, 10, 1C, 214, 28, 1, 1 +root - INFO - 32188: A5, 10, 18, 188, 29, 1, 1 +root - INFO - 32386: A5, 10, 15, 524, 29, 1, 1 +root - INFO - 32920: A5, 10, 1C, 214, 29, 1, 1 +root - INFO - 33144: A5, 10, 18, 188, 30, 1, 1 +root - INFO - 33342: A5, 10, 15, 524, 30, 1, 1 +root - INFO - 33876: A5, 10, 1C, 214, 30, 1, 1 +root - INFO - 34100: A5, 10, 18, 188, 31, 1, 1 +root - INFO - 34298: A5, 10, 15, 524, 31, 1, 1 +root - INFO - 34832: A5, 10, 1C, 214, 31, 1, 1 +root - INFO - 35056: A5, 10, 18, 188, 32, 1, 1 +root - INFO - 35254: A5, 10, 15, 524, 32, 1, 1 +root - INFO - 35788: A5, 10, 1C, 214, 32, 1, 1 +root - INFO - 36012: A5, 10, 18, 188, 33, 1, 1 +root - INFO - 36210: A5, 10, 15, 524, 33, 1, 1 diff --git a/examples/data/dolfyn/test_data/Sig500_Echo.nc b/examples/data/dolfyn/test_data/Sig500_Echo.nc index 78dbdd06..9a78c792 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_Echo.nc and b/examples/data/dolfyn/test_data/Sig500_Echo.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_Echo_clean.nc b/examples/data/dolfyn/test_data/Sig500_Echo_clean.nc index d6f18711..fcb414cd 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_Echo_clean.nc and b/examples/data/dolfyn/test_data/Sig500_Echo_clean.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_Echo_crop.nc b/examples/data/dolfyn/test_data/Sig500_Echo_crop.nc index 80d4bca8..bd020aa6 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_Echo_crop.nc and b/examples/data/dolfyn/test_data/Sig500_Echo_crop.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_Echo_earth2inst.nc b/examples/data/dolfyn/test_data/Sig500_Echo_earth2inst.nc index b44b1d8c..65f7b833 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_Echo_earth2inst.nc and b/examples/data/dolfyn/test_data/Sig500_Echo_earth2inst.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_Echo_inst2beam.nc b/examples/data/dolfyn/test_data/Sig500_Echo_inst2beam.nc index 8e12197d..4505bc2f 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_Echo_inst2beam.nc and b/examples/data/dolfyn/test_data/Sig500_Echo_inst2beam.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_dp_ice1.nc b/examples/data/dolfyn/test_data/Sig500_dp_ice1.nc index 2ac223c1..ac621054 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_dp_ice1.nc and b/examples/data/dolfyn/test_data/Sig500_dp_ice1.nc differ diff --git a/examples/data/dolfyn/test_data/Sig500_last_ensemble_is_whole.nc b/examples/data/dolfyn/test_data/Sig500_last_ensemble_is_whole.nc index f52d8df4..48bd0a5b 100644 Binary files a/examples/data/dolfyn/test_data/Sig500_last_ensemble_is_whole.nc and b/examples/data/dolfyn/test_data/Sig500_last_ensemble_is_whole.nc differ diff --git a/examples/data/dolfyn/test_data/Sig_SkippedPings01.nc b/examples/data/dolfyn/test_data/Sig_SkippedPings01.nc index 90b41d3c..bb516fbf 100644 Binary files a/examples/data/dolfyn/test_data/Sig_SkippedPings01.nc and b/examples/data/dolfyn/test_data/Sig_SkippedPings01.nc differ diff --git a/examples/data/dolfyn/test_data/VelEchoBT01.nc b/examples/data/dolfyn/test_data/VelEchoBT01.nc index f2fb6183..38bda934 100644 Binary files a/examples/data/dolfyn/test_data/VelEchoBT01.nc and b/examples/data/dolfyn/test_data/VelEchoBT01.nc differ diff --git a/examples/data/dolfyn/test_data/VelEchoBT01_rotate_beam2inst.nc b/examples/data/dolfyn/test_data/VelEchoBT01_rotate_beam2inst.nc index 6cefd020..747b258f 100644 Binary files a/examples/data/dolfyn/test_data/VelEchoBT01_rotate_beam2inst.nc and b/examples/data/dolfyn/test_data/VelEchoBT01_rotate_beam2inst.nc differ diff --git a/mhkit/dolfyn/io/base.py b/mhkit/dolfyn/io/base.py index 6326f6e0..6a90c65f 100644 --- a/mhkit/dolfyn/io/base.py +++ b/mhkit/dolfyn/io/base.py @@ -323,11 +323,8 @@ def _create_dataset(data): else: # amp, corr, prcnt_gd, status dim0 = "beam" - if not any(val in key for val in tag) or ("_avg" in key): - if "_avg" in key: - tg = "_avg" - else: - tg = "" + if not any(val in key for val in tag) or (tg in ["_avg", "_bt"]): + tg = "" if tg not in ["_avg", "_bt"] else tg ds_dict[key] = { "dims": (dim0, "range" + tg, "time" + tg), "data": data["data_vars"][key], diff --git a/mhkit/dolfyn/io/nortek2.py b/mhkit/dolfyn/io/nortek2.py index a536bca5..80fb489e 100644 --- a/mhkit/dolfyn/io/nortek2.py +++ b/mhkit/dolfyn/io/nortek2.py @@ -81,10 +81,8 @@ def read_signature( ) d = rdr.readfile(nens[0], nens[1]) rdr.sci_data(d) - if rdr._dp: - _clean_dp_skips(d) - out = _reorg(d) - _reduce(out) + _clean_skips(d) + out = _reduce(_reorg(d)) # Convert time to dt64 and fill gaps coords = out["coords"] @@ -246,7 +244,7 @@ def _init_burst_readers( ) elif rdr_id == 23: self._burst_readers[rdr_id] = defs._calc_bt_struct( - cfg["_config"], cfg["n_beams"] + cfg["_config"], cfg["n_beams"], cfg["n_cells"], cfg["_d_ver"] ) else: self._burst_readers[rdr_id] = defs._calc_burst_struct( @@ -301,9 +299,11 @@ def _read_str(self, size): string = string[1:-1] return id, string - def _read_burst(self, id, dat, c, echo=False): + def _read_burst(self, id, dat, c, sz, echo=False): rdr = self._burst_readers[id] - rdr.read_into(self.f, dat, c) + to_skip = rdr.read_into(self.f, dat, c, sz) + if to_skip: + self.f.seek(to_skip, 1) def readfile(self, ens_start=0, ens_stop=None): # If the lastblock is not whole, we don't read it. @@ -326,10 +326,13 @@ def readfile(self, ens_start=0, ens_stop=None): except IOError: return outdat id = hdr["id"] - if id in [21, 22, 23, 24, 28]: # "burst data record" (vel + ast), + if id not in outdat: + # If the ID was skipped in the indexer, it won't be in outdat + self.f.seek(hdr["sz"], 1) + elif id in [21, 22, 23, 24, 28]: # "burst data record" (vel + ast), # "avg data record" (vel_avg + ast_avg), "bottom track data record" (bt), # "interleaved burst data record" (vel_b5), "echosounder record" (echo) - self._read_burst(id, outdat[id], c) + self._read_burst(id, outdat[id], c, hdr["sz"]) elif id in [26, 31]: # "burst altimeter raw record" (_altraw), "avg altimeter raw record" (_altraw_avg) rdr = self._burst_readers[id] @@ -367,7 +370,7 @@ def readfile(self, ens_start=0, ens_stop=None): "The number of samples in this 'Altimeter Raw' " "burst is different from prior bursts." ) - self._read_burst(id, outdat[id], c_altraw[id]) + self._read_burst(id, outdat[id], c_altraw[id], hdr["sz"]) outdat[id]["ensemble"][c_altraw[id]] = c c_altraw[id] += 1 @@ -506,10 +509,6 @@ def _reorg(dat): (28, "_echo"), (31, "raw_avg"), ]: - if id in [24, 26]: - collapse_exclude = [0] - else: - collapse_exclude = [] if id not in dat: continue dnow = dat[id] @@ -520,7 +519,8 @@ def _reorg(dat): dnow["standard_name"].pop(ky) outdat["standard_name"].update(dnow["standard_name"]) cfg["burst_config" + tag] = lib._headconfig_int2dict( - lib._collapse(dnow["config"], exclude=collapse_exclude, name="config") + lib._collapse(dnow["config"], exclude=[0], name="config"), + mode="bt" if "bt" in tag else "burst", ) outdat["coords"]["time" + tag] = lib._calc_time( dnow["year"] + np.uint16(1900), @@ -531,34 +531,30 @@ def _reorg(dat): dnow["second"], dnow["usec100"].astype("uint32") * 100, ) - tmp = lib._beams_cy_int2dict( - lib._collapse( - dnow["beam_config"], exclude=collapse_exclude, name="beam_config" - ), - 21, # always 21 here - ) + + d_ver = lib._collapse(dnow["ver"], exclude=[0], name="ver") + beams_cy = lib._collapse(dnow["beam_config"], exclude=[0], name="beam_config") + # Deal with data version 9 for bottom track + if (id == 23) and (d_ver >= 9): + n_cells = lib._collapse(dnow["n_cells"], exclude=[0], name="n_cells") + beams_cy = lib._repack_bt_beams_cy(beams_cy, n_cells) + + # Unpack cells, coordinate axes and beams + tmp = lib._beams_cy_int2dict(beams_cy, id, d_ver) cfg["n_cells" + tag] = int(tmp["n_cells"]) cfg["coord_sys_axes" + tag] = tmp["cy"] cfg["n_beams" + tag] = int(tmp["n_beams"]) - cfg["ambig_vel" + tag] = round( - float(lib._collapse(dnow["ambig_vel"], name="ambig_vel")), 3 - ) - for ky in [ - "SerialNum", - "nominal_corr", - ]: - cfg[ky + tag] = int( - lib._collapse(dnow[ky], exclude=collapse_exclude, name=ky) + cfg["SerialNum" + tag] = int( + lib._collapse(dnow["SerialNum"], exclude=[0], name="SerialNum") + ) + for ky in ["cell_size", "power_level_dB", "ambig_vel"]: + cfg[ky + tag] = round( + float(lib._collapse(dnow[ky], exclude=[0], name=ky)), 3 ) - - for ky in [ - "cell_size", - "blank_dist", - "power_level_dB", - ]: - cfg[ky + tag] = float( - lib._collapse(dnow[ky], exclude=collapse_exclude, name=ky) + if id != 23: + cfg["blank_dist" + tag] = float( + lib._collapse(dnow["blank_dist"], exclude=[0], name="blank_dist") ) for ky in [ @@ -674,9 +670,7 @@ def _reorg(dat): cfg["rotate_vars"] = [] cy = cfg["coord_sys_axes_avg"] else: - cfg["rotate_vars"] = [ - "vel", - ] + cfg["rotate_vars"] = ["vel"] cy = cfg["coord_sys_axes"] outdat["attrs"]["coord_sys"] = {"XYZ": "inst", "ENU": "earth", "beam": "beam"}[cy] @@ -693,10 +687,9 @@ def _reorg(dat): return outdat -def _clean_dp_skips(data): +def _clean_skips(data): """ - Removes zeros from interwoven measurements taken in a dual profile - configuration. + Removes zeros from interwoven measurements """ for id in data: @@ -724,14 +717,6 @@ def _reduce(data): dc = data["coords"] da = data["attrs"] - # Average these fields - for ky in ["c_sound", "temp", "pressure", "temp_press", "temp_clock", "batt"]: - lib._reduce_by_average(dv, ky, ky + "_b5") - - # Angle-averaging is treated separately - for ky in ["heading", "pitch", "roll"]: - lib._reduce_by_average_angle(dv, ky, ky + "_b5") - if "vel" in dv: dc["range"] = (np.arange(dv["vel"].shape[1]) + 1) * da["cell_size"] + da[ "blank_dist" @@ -784,6 +769,8 @@ def _reduce(data): tm[irow, icol] = tmat["M" + str(irow + 1) + str(icol + 1)] dv["beam2inst_orientmat"] = tm + return data + def split_dp_datasets(ds): """ diff --git a/mhkit/dolfyn/io/nortek2_defs.py b/mhkit/dolfyn/io/nortek2_defs.py index 56cc0e05..08713a68 100644 --- a/mhkit/dolfyn/io/nortek2_defs.py +++ b/mhkit/dolfyn/io/nortek2_defs.py @@ -71,13 +71,14 @@ def init_data(self, npings): out[nm] = _nans(shp + [npings], dtype=np.dtype(fmt[0])) return out - def read_into(self, fobj, data, ens, cs=None): - dat_tuple = self.read(fobj, cs=cs) + def read_into(self, fobj, data, ens, sz, cs=None): + dat_tuple, to_skip = self.read(fobj, sz=sz, cs=cs) for nm, shp, d in zip(self._names, self._shape, dat_tuple): try: data[nm][..., ens] = d except ValueError: data[nm][..., ens] = np.asarray(d).reshape(shp) + return to_skip @property def format( @@ -85,8 +86,16 @@ def format( ): return _format(self._format, self._N) - def read(self, fobj, cs=None): + def read(self, fobj, sz=None, cs=None): bytes = fobj.read(self.nbyte) + # For newer versions of id data packets + if sz is not None: + extra_bytes = sz - self.nbyte + if extra_bytes < 0: + raise ValueError( + "Data definition contains different number of bytes than " + f"data packet: {extra_bytes}" + ) if len(bytes) != self.nbyte: raise IOError("End of file.") data = self._struct.unpack(bytes) @@ -110,7 +119,10 @@ def read(self, fobj, cs=None): else: out.append(data[c : (c + n)]) c += n - return out + if sz is not None: + return out, extra_bytes + else: + return out def read2dict(self, fobj, cs=False): return {self._names[idx]: dat for idx, dat in enumerate(self.read(fobj, cs=cs))} @@ -291,98 +303,129 @@ def __call__(self, array): ("_ensemble", "I", [], None), ] -_bt_hdr = [ - ("ver", "B", [], None), - ("DatOffset", "B", [], None), - ("config", "H", [], None), - ("SerialNum", "I", [], None), - ("year", "B", [], None), - ("month", "B", [], None), - ("day", "B", [], None), - ("hour", "B", [], None), - ("minute", "B", [], None), - ("second", "B", [], None), - ("usec100", "H", [], None), - ( - "c_sound", - "H", - [], - _LinFunc(0.1, dtype=dt32), - "m s-1", - "Speed of Sound", - "speed_of_sound_in_sea_water", - ), - ( - "temp", - "h", - [], - _LinFunc(0.01, dtype=dt32), - "degree_C", - "Temperature", - "sea_water_temperature", - ), - ( - "pressure", - "I", - [], - _LinFunc(0.001, dtype=dt32), - "dbar", - "Pressure", - "sea_water_pressure", - ), - ( - "heading", - "H", - [], - _LinFunc(0.01, dtype=dt32), - "degree", - "Heading", - "platform_orientation", - ), - ("pitch", "h", [], _LinFunc(0.01, dtype=dt32), "degree", "Pitch", "platform_pitch"), - ("roll", "h", [], _LinFunc(0.01, dtype=dt32), "degree", "Roll", "platform_roll"), - ("beam_config", "H", [], None), - ("cell_size", "H", [], _LinFunc(0.001), "m"), - ("blank_dist", "H", [], _LinFunc(0.01), "m"), - ("nominal_corr", "B", [], None, "%"), - ("unused", "B", [], None), - ( - "batt", - "H", - [], - _LinFunc(0.1, dtype=dt32), - "V", - "Battery Voltage", - "battery_voltage", - ), - ("mag", "h", [3], None, "uT", "Compass"), - ( - "accel", - "h", - [3], - _LinFunc(1.0 / 16384 * grav, dtype=dt32), - "m s-2", - "Acceleration", - "", - ), - ("ambig_vel", "I", [], _LinFunc(0.001, dtype=dt32), "m s-1"), - ("data_desc", "H", [], None), - ("xmit_energy", "H", [], None, "dB", "Sound Pressure Level of Acoustic Signal"), - ("vel_scale", "b", [], None), - ("power_level_dB", "b", [], _LinFunc(dtype=dt32), "dB"), - ("temp_mag", "h", [], None), # uncalibrated - ( - "temp_clock", - "h", - [], - _LinFunc(0.01, dtype=dt32), - "degree_C", - "Internal Clock Temperature", - ), - ("error", "I", [], None, "1", "Error Code"), - ("status", "I", [], None, "1", "Status Code"), - ("_ensemble", "I", [], None), -] + +def _bt_hdr(ver): + if ver == 9: # version 9 + options = [ + ("beam_config", "B", [], None), + ("pad1", "B", [], None), + ("n_cells", "H", [], None), + ("cell_size", "H", [], _LinFunc(0.001), "m"), + ("unused", "B", [], None), + ("temp_pres", "B", [], None), + ] + else: # version 1 + options = [ + ("beam_config", "H", [], None), + ("cell_size", "H", [], _LinFunc(0.001), "m"), + ("blank_dist", "H", [], None), + ("vel_range", "H", [], None), + ] + out = [ + ("ver", "B", [], None), + ("DatOffset", "B", [], None), + ("config", "H", [], None), + ("SerialNum", "L", [], None), + ("year", "B", [], None), + ("month", "B", [], None), + ("day", "B", [], None), + ("hour", "B", [], None), + ("minute", "B", [], None), + ("second", "B", [], None), + ("usec100", "H", [], None), + ( + "c_sound", + "H", + [], + _LinFunc(0.1, dtype=dt32), + "m s-1", + "Speed of Sound", + "speed_of_sound_in_sea_water", + ), + ( + "temp", + "h", + [], + _LinFunc(0.01, dtype=dt32), + "degree_C", + "Temperature", + "sea_water_temperature", + ), + ( + "pressure", + "L", + [], + _LinFunc(0.001, dtype=dt32), + "dbar", + "Pressure", + "sea_water_pressure", + ), + ( + "heading", + "H", + [], + _LinFunc(0.01, dtype=dt32), + "degree", + "Heading", + "platform_orientation", + ), + ( + "pitch", + "h", + [], + _LinFunc(0.01, dtype=dt32), + "degree", + "Pitch", + "platform_pitch", + ), + ( + "roll", + "h", + [], + _LinFunc(0.01, dtype=dt32), + "degree", + "Roll", + "platform_roll", + ), + *options, + ( + "batt", + "H", + [], + _LinFunc(0.1, dtype=dt32), + "V", + "Battery Voltage", + "battery_voltage", + ), + ("mag", "h", [3], None, "uT", "Compass"), + ( + "accel", + "h", + [3], + _LinFunc(1.0 / 16384 * grav, dtype=dt32), + "m s-2", + "Acceleration", + "", + ), + ("ambig_vel", "I", [], _LinFunc(0.001, dtype=dt32), "m s-1"), + ("data_desc", "H", [], None), + ("xmit_energy", "H", [], None, "dB", "Sound Pressure Level of Acoustic Signal"), + ("vel_scale", "b", [], None), + ("power_level_dB", "b", [], _LinFunc(dtype=dt32), "dB"), + ("temp_mag", "h", [], None), # uncalibrated + ( + "temp_clock", + "h", + [], + _LinFunc(0.01, dtype=dt32), + "degree_C", + "Internal Clock Temperature", + ), + ("error", "I", [], None, "1", "Error Code"), + ("status", "I", [], None, "1", "Status Code"), + ("_ensemble", "I", [], None), + ] + return out _ahrs_def = [ @@ -399,29 +442,51 @@ def __call__(self, array): ] -def _calc_bt_struct(config, nb): +def _calc_bt_struct(config, nb, nc, d_ver=1): flags = lib._headconfig_int2dict(config, mode="bt") - dd = copy(_bt_hdr) - if flags["vel"]: - # units handled in Ad2cpReader.sci_data - dd.append( - ("vel", "i", [nb], None, "m s-1", "Platform Velocity from Bottom Track") - ) - if flags["dist"]: - dd.append( - ( - "dist", - "i", - [nb], - _LinFunc(0.001, dtype=dt32), - "m", - "Bottom Track Measured Depth", + dd = _bt_hdr(d_ver) + + if d_ver == 1: + if flags["vel"]: + # units handled in Ad2cpReader.sci_data + dd.append( + ("vel", "i", [nb], None, "m s-1", "Platform Velocity from Bottom Track") ) - ) - if flags["fom"]: - dd.append(("fom", "H", [nb], None, "1", "Figure of Merit")) - if flags["ahrs"]: - dd += _ahrs_def + if flags["dist"]: + dd.append( + ( + "dist", + "i", + [nb], + _LinFunc(0.001, dtype=dt32), + "m", + "Bottom Track Measured Depth", + ) + ) + if flags["fom"]: + dd.append(("fom", "H", [nb], None, "1", "Figure of Merit")) + if flags["ahrs"]: + dd += _ahrs_def + else: # Data record version 9 + if flags["fom"]: + dd.append(("fom", "H", [nb], None, "1", "Figure of Merit")) + if flags["vel"]: + # units handled in Ad2cpReader.sci_data + dd.append( + ("vel", "i", [nb], None, "m s-1", "Platform Velocity from Bottom Track") + ) + if flags["dist"]: + dd.append( + ( + "dist", + "i", + [nb], + _LinFunc(0.001, dtype=dt32), + "m", + "Bottom Track Measured Depth", + ) + ) + # Does not appear to contain AHRS data even if flag says so return _DataDef(dd) diff --git a/mhkit/dolfyn/io/nortek2_lib.py b/mhkit/dolfyn/io/nortek2_lib.py index 7c1f17ee..63374f56 100644 --- a/mhkit/dolfyn/io/nortek2_lib.py +++ b/mhkit/dolfyn/io/nortek2_lib.py @@ -7,37 +7,6 @@ from .. import time from .base import _abspath - -def _reduce_by_average(data, ky0, ky1): - # Average two arrays together, if they both exist. - if ky1 in data: - tmp = data.pop(ky1) - if ky0 in data: - data[ky0] += tmp - data[ky0] = data[ky0] / 2 - else: - data[ky0] = tmp - - -def _reduce_by_average_angle(data, ky0, ky1, degrees=True): - # Average two arrays of angles together, if they both exist. - if degrees: - rad_fact = np.pi / 180 - else: - rad_fact = 1 - if ky1 in data: - if ky0 in data: - data[ky0] = ( - np.angle( - np.exp(1j * data.pop(ky0) * rad_fact) - + np.exp(1j * data.pop(ky1) * rad_fact) - ) - / rad_fact - ) - else: - data[ky0] = data.pop(ky1) - - # This is the data-type of the index file. # This must match what is written-out by the create_index function. _index_version = 1 @@ -152,23 +121,6 @@ def _create_index(infile, outfile, init_pos, eof, debug): N = dict.fromkeys(ids, 0) config = 0 last_ens = dict.fromkeys(ids, -1) - seek_2ens = { - 21: 40, # 0x15 burst - 22: 40, # 0x16 average - 23: 42, # 0x17 bottom track, starts from "42" - 24: 40, # 0x18 interleaved burst (beam 5) - 26: 40, # 0x1A burst altimeter - 27: 40, # 0x1B DVL bottom track - 28: 40, # 0x1C echo sounder - 29: 40, # 0x1D DVL water track - 30: 40, # 0x1E altimeter - 31: 40, # 0x1F avg altimeter - 35: 40, # 0x23 raw echo sounder - 36: 40, # 0x24 raw tx echo sounder - 48: 40, # 0x30 processed wave - # 160: 40, # 0xA0 string (header info, GPS NMEA data) - # 192: 40, # 0xC0 Nortek Data format 8 record - } pos = 0 header_check_flag = 0 # leave room for header plus other data (12 + 76) @@ -176,7 +128,7 @@ def _create_index(infile, outfile, init_pos, eof, debug): if init_pos and not pos: fin.seek(init_pos, 1) try: - dat = struct.unpack(" 0: - if (ens[idk] == 1) or (ens[idk] < last_ens[idk]): - # Covers all id keys saved in "burst mode" - # Covers ID keys not saved in sequential order - ens[idk] = last_ens[idk] + 1 - - if last_ens[idk] > 0 and last_ens[idk] != ens[idk]: N[idk] += 1 + # Update our last ensemble before writing to index file + last_ens[idk] = ens[idk] + # Skip to next ID number + fin.seek(dat[4] - d_off, 1) + # Write to index file fout.write( struct.pack( " N_id)[0] + ibad = np.nonzero(diff > N_id)[0] # Check if spacing is equal for dual profiling ADCPs if dp: @@ -326,11 +306,12 @@ def _check_index(idx, infile, fix_hw_ens=False, dp=False): for ib in ibad: # The ping number reported here may not be quite right if # the ensemble count is wrong. - logging.warning( - "Skipped ping (ID: {}) in file {} at ensemble {}.".format( - id, infile, idx["ens"][inds[ib + 1] - 1] - ) + msg = ( + f"Skipped ping (ID: {hex(id)}) in file {infile} at ensemble " + f"{idx['ens'][inds[ib + 1] - 1]}." ) + warnings.warn(msg) + logging.warning(msg) hwe[inds[(ib + 1) :]] += 1 ens[inds[(ib + 1) :]] += 1 @@ -488,7 +469,7 @@ def _headconfig_int2dict(val, mode="burst"): temp_valid=_getbit(val, 1), compass_valid=_getbit(val, 2), tilt_valid=_getbit(val, 3), - # bit 4 is unused + # ext_sensor=_getbit(val, 4), vel=_getbit(val, 5), amp=_getbit(val, 6), corr=_getbit(val, 7), @@ -499,7 +480,7 @@ def _headconfig_int2dict(val, mode="burst"): ahrs=_getbit(val, 12), p_gd=_getbit(val, 13), std=_getbit(val, 14), - # bit 15 is unused + # spec=_getbit(val, 15), ) elif mode == "bt": return dict( @@ -509,11 +490,13 @@ def _headconfig_int2dict(val, mode="burst"): tilt_valid=_getbit(val, 3), # bit 4 is unused vel=_getbit(val, 5), - # bits 6-7 unused + amp=_getbit(val, 6), + corr=_getbit(val, 7), dist=_getbit(val, 8), fom=_getbit(val, 9), ahrs=_getbit(val, 10), - # bits 10-15 unused + aux=_getbit(val, 11), + # bits 12-15 unused ) @@ -565,12 +548,28 @@ def _alt_status2data(val): return out -def _beams_cy_int2dict(val, id): +def _repack_bt_beams_cy(beams_cy, n_cells): + # "beams_cy": < bit 7-5: Number of beams, bit 4-3: coordinate system + n_beams = (beams_cy >> 5) & 0b111 + cy_idx = (beams_cy >> 3) & 0b11 + return (n_cells << 5) | (n_beams << 2) | cy_idx + + +def _beams_cy_int2dict(val, id, d_ver=3): """Convert the beams/coordinate-system bytes to a dict of values.""" if id == 28: # 0x1C (echosounder) - return dict(n_cells=val) - elif id in [26, 31]: + return dict(n_cells=val, cy="beam", n_beams=1) + elif id in [26, 31]: # Altimeter return dict(n_cells=val & (2**10 - 1), cy="beam", n_beams=1) + elif (id == 23) and (d_ver == 9): + # Unpack custom beams_cy for bottom track data version 9 + return dict( + n_cells=val >> 5, + cy=["ENU", "XYZ", "beam", None][(val & 31) & 3], + n_beams=((val & 31) >> 2) & 7, # usually = 4 + ) + # Data definition version 1 and 3 + # "ncells from bits 9-0, nbeams from bits 15-12, and coordinate system from bits 11-10". return dict( n_cells=val & (2**10 - 1), cy=["ENU", "XYZ", "beam", None][val >> 10 & 3], @@ -640,6 +639,7 @@ def _calc_config(index): inds = index["ID"] == id _config = index["config"][inds] _beams_cy = index["beams_cy"][inds] + _data_ver = index["d_ver"][inds] # Check that these variables are consistent if not _isuniform(_config): @@ -649,7 +649,14 @@ def _calc_config(index): if id == 23: # change in "n_cells" doesn't matter lob = np.unique(_beams_cy) - beams = list(map(_beams_cy_int2dict, lob, 23 * np.ones(lob.size))) + beams = list( + map( + _beams_cy_int2dict, + lob, + 23 * np.ones(lob.size), + _data_ver[0] * np.ones(lob.size), + ) + ) if all([d["cy"] for d in beams]) and all([d["n_beams"] for d in beams]): err = False if err: @@ -657,9 +664,10 @@ def _calc_config(index): # Now that we've confirmed they are the same: config[id] = _headconfig_int2dict(_config[0], mode=type) - config[id].update(_beams_cy_int2dict(_beams_cy[0], id)) + config[id].update(_beams_cy_int2dict(_beams_cy[0], id, _data_ver[0])) config[id]["_config"] = _config[0] config[id]["_beams_cy"] = _beams_cy[0] + config[id]["_d_ver"] = _data_ver[0] config[id]["type"] = type config[id].pop("cy", None) diff --git a/mhkit/tests/dolfyn/test_api.py b/mhkit/tests/dolfyn/test_api.py index 272ff121..fc44dcd6 100644 --- a/mhkit/tests/dolfyn/test_api.py +++ b/mhkit/tests/dolfyn/test_api.py @@ -1,7 +1,6 @@ from .base import load_netcdf as load, rfnm import unittest - make_data = False vec = load("vector_data01.nc") sig = load("BenchFile01.nc") @@ -9,7 +8,7 @@ class api_testcase(unittest.TestCase): - def test_repr(self): + def test_repr(self, make_data=False): _str = [] for dat, fnm in [ (vec, rfnm("vector_data01.repr.txt")), @@ -17,10 +16,12 @@ def test_repr(self): (rdi, rfnm("RDI_test01.repr.txt")), ]: _str = dat.velds.__repr__() + if make_data: with open(fnm, "w") as fl: fl.write(_str) - else: - with open(fnm, "r") as fl: - test_str = fl.read() - assert test_str == _str + return + + with open(fnm, "r") as fl: + test_str = fl.read() + assert test_str == _str diff --git a/mhkit/tests/dolfyn/test_read_adp.py b/mhkit/tests/dolfyn/test_read_adp.py index 1ac29179..e1af64d4 100644 --- a/mhkit/tests/dolfyn/test_read_adp.py +++ b/mhkit/tests/dolfyn/test_read_adp.py @@ -226,9 +226,9 @@ def test_io_nortek2(self): assert_allclose(td_sig_raw_avg, dat_sig_raw_avg, atol=1e-6) assert_allclose(td_sig_avg, dat_sig_avg, atol=1e-6) assert_allclose(td_sig_rt, dat_sig_rt, atol=1e-6) - assert_allclose(td_sig5_leiw, dat_sig5_leiw, atol=1e-6) assert_allclose(td_sig_skip, dat_sig_skip, atol=1e-6) assert_allclose(td_sig_badt, dat_sig_badt, atol=1e-6) + assert_allclose(td_sig5_leiw, dat_sig5_leiw, atol=1e-6) assert_allclose(td_sig_dp1_all, dat_sig_dp1_all, atol=1e-6) assert_allclose(td_sig_dp1_ice, dat_sig_dp1_ice, atol=1e-6) assert_allclose(td_sig_dp2_echo, dat_sig_dp2_echo, atol=1e-6) diff --git a/mhkit/tests/dolfyn/test_rotate_adp.py b/mhkit/tests/dolfyn/test_rotate_adp.py index 9b25d643..ad7edb31 100644 --- a/mhkit/tests/dolfyn/test_rotate_adp.py +++ b/mhkit/tests/dolfyn/test_rotate_adp.py @@ -86,7 +86,7 @@ def test_rotate_inst2earth(self): save(tdwr2, "winriver02_rotate_ship2earth.nc") save(td_sig, "BenchFile01_rotate_inst2earth.nc") save(td_sig_i, "Sig1000_IMU_rotate_inst2earth.nc") - save(td_sig_ie, "Sig500_Echo_earth2inst.nc") + return td_awac = rotate2(load("AWAC_test01_earth2inst.nc"), "earth", inplace=False)