Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions imap_processing/glows/l1a/glows_l1a.py
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,15 @@ def generate_histogram_dataset(
)
hist_l1a_list = valid_hists

# Filter out histograms with no recorded events (all-zero histogram data).
valid_hists = [hist for hist in hist_l1a_list if hist.number_of_events > 0]
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did same in the past for GLOWS and feel like it didn't catch all cases of invalid histogram. I hope checking hist.number_of_events > 0 is enough to filter out valid histogram. @maxinelasp may be better help here.

if len(valid_hists) < len(hist_l1a_list):
logger.warning(
f"GLOWS: Filtered out {len(hist_l1a_list) - len(valid_hists)} "
f"histogram(s) with number_of_events == 0."
)
hist_l1a_list = valid_hists

# Store timestamps for each HistogramL1A object.
time_data: np.ndarray = np.zeros(len(hist_l1a_list), dtype=np.int64)
# Data in lists, for each of the 25 time varying datapoints in HistogramL1A
Expand Down
16 changes: 16 additions & 0 deletions imap_processing/tests/glows/test_glows_l1a_cdf.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,22 @@ def test_generate_histogram_dataset_filters_zero_imap_start_time(l1a_test_data):
assert len(dataset["epoch"].values) == 2


def test_generate_histogram_dataset_filters_zero_events(l1a_test_data):
histogram_l1a, _ = l1a_test_data
glows_attrs = create_glows_attr_obj()

zero_event_hist = MagicMock()
zero_event_hist.number_of_bins_per_histogram = 3600
zero_event_hist.imap_start_time = histogram_l1a[0].imap_start_time
zero_event_hist.number_of_events = 0

mixed_list = [zero_event_hist, histogram_l1a[0], zero_event_hist, histogram_l1a[1]]

dataset = generate_histogram_dataset(mixed_list, glows_attrs)

assert len(dataset["epoch"].values) == 2


def test_generate_de_dataset(l1a_test_data):
_, de_l1a = l1a_test_data
glows_attrs = create_glows_attr_obj()
Expand Down
Loading