Loading...
Searching...
No Matches
BitMaskedArray.h
Go to the documentation of this file.
1// BSD 3-Clause License; see https://github.com/scikit-hep/awkward-1.0/blob/main/LICENSE
2
3#ifndef AWKWARD_BITMASKEDARRAY_H_
4#define AWKWARD_BITMASKEDARRAY_H_
5
6#include <string>
7#include <memory>
8#include <vector>
9
10#include "awkward/common.h"
11#include "awkward/Slice.h"
12#include "awkward/Index.h"
13#include "awkward/Content.h"
14
15namespace awkward {
16 template <typename T, bool ISOPTION>
17 class IndexedArrayOf;
18
19 class ByteMaskedArray;
20
25 public:
27 BitMaskedForm(bool has_identities,
28 const util::Parameters& parameters,
29 const FormKey& form_key,
30 Index::Form mask,
31 const FormPtr& content,
32 bool valid_when,
33 bool lsb_order);
34
36 mask() const;
37
38 const FormPtr
39 content() const;
40
41 bool
42 valid_when() const;
43
44 bool
45 lsb_order() const;
46
47 const TypePtr
48 type(const util::TypeStrs& typestrs) const override;
49
50 void
51 tojson_part(ToJson& builder, bool verbose) const override;
52
53 const FormPtr
54 shallow_copy() const override;
55
56 const FormPtr
57 with_form_key(const FormKey& form_key) const override;
58
59 const std::string
60 purelist_parameter(const std::string& key) const override;
61
62 bool
63 purelist_isregular() const override;
64
65 int64_t
66 purelist_depth() const override;
67
68 bool
69 dimension_optiontype() const override;
70
71 const std::pair<int64_t, int64_t>
72 minmax_depth() const override;
73
74 const std::pair<bool, int64_t>
75 branch_depth() const override;
76
77 int64_t
78 numfields() const override;
79
80 int64_t
81 fieldindex(const std::string& key) const override;
82
83 const std::string
84 key(int64_t fieldindex) const override;
85
86 bool
87 haskey(const std::string& key) const override;
88
89 const std::vector<std::string>
90 keys() const override;
91
92 bool
93 istuple() const override;
94
95 bool
96 equal(const FormPtr& other,
97 bool check_identities,
98 bool check_parameters,
99 bool check_form_key,
100 bool compatibility_check) const override;
101
102 virtual const FormPtr
103 getitem_range() const override;
104
105 const FormPtr
106 getitem_field(const std::string& key) const override;
107
108 const FormPtr
109 getitem_fields(const std::vector<std::string>& keys) const override;
110
111 const FormPtr
113
114 private:
115 Index::Form mask_;
116 const FormPtr content_;
117 bool valid_when_;
118 bool lsb_order_;
119 };
120
128 public:
155 BitMaskedArray(const IdentitiesPtr& identities,
156 const util::Parameters& parameters,
157 const IndexU8& mask,
158 const ContentPtr& content,
159 bool valid_when,
160 int64_t length,
161 bool lsb_order);
162
169 const IndexU8
170 mask() const;
171
174 const ContentPtr
175 content() const;
176
181 bool
182 valid_when() const;
183
190 bool
191 lsb_order() const;
192
195 const ContentPtr
196 project() const;
197
202 const ContentPtr
203 project(const Index8& mask) const;
204
208 const Index8
209 bytemask() const;
210
216 const ContentPtr
218
220 const std::shared_ptr<ByteMaskedArray>
222
225 const std::shared_ptr<IndexedArrayOf<int64_t, true>>
227
229 const std::string
230 classname() const override;
231
232 void
233 setidentities() override;
234
235 void
236 setidentities(const IdentitiesPtr& identities) override;
237
238 const TypePtr
239 type(const util::TypeStrs& typestrs) const override;
240
241 const FormPtr
242 form(bool materialize) const override;
243
245 kernels() const override;
246
247 void
248 caches(std::vector<ArrayCachePtr>& out) const override;
249
250 const std::string
251 tostring_part(const std::string& indent,
252 const std::string& pre,
253 const std::string& post) const override;
254
255 void
256 tojson_part(ToJson& builder, bool include_beginendlist) const override;
257
258 void
259 nbytes_part(std::map<size_t, int64_t>& largest) const override;
260
264 int64_t
265 length() const override;
266
267 const ContentPtr
268 shallow_copy() const override;
269
270 const ContentPtr
271 deep_copy(bool copyarrays,
272 bool copyindexes,
273 bool copyidentities) const override;
274
275 void
276 check_for_iteration() const override;
277
278 const ContentPtr
279 getitem_nothing() const override;
280
281 const ContentPtr
282 getitem_at(int64_t at) const override;
283
284 const ContentPtr
285 getitem_at_nowrap(int64_t at) const override;
286
287 const ContentPtr
288 getitem_range(int64_t start, int64_t stop) const override;
289
290 const ContentPtr
291 getitem_range_nowrap(int64_t start, int64_t stop) const override;
292
293 const ContentPtr
294 getitem_field(const std::string& key) const override;
295
296 const ContentPtr
297 getitem_field(const std::string& key,
298 const Slice& only_fields) const override;
299
300 const ContentPtr
301 getitem_fields(const std::vector<std::string>& keys) const override;
302
303 const ContentPtr
304 getitem_fields(const std::vector<std::string>& keys,
305 const Slice& only_fields) const override;
306
307 const ContentPtr
309 const Slice& tail,
310 const Index64& advanced) const override;
311
312 const ContentPtr
313 carry(const Index64& carry, bool allow_lazy) const override;
314
315 int64_t
316 purelist_depth() const override;
317
318 const std::pair<int64_t, int64_t>
319 minmax_depth() const override;
320
321 const std::pair<bool, int64_t>
322 branch_depth() const override;
323
324 int64_t
325 numfields() const override;
326
327 int64_t
328 fieldindex(const std::string& key) const override;
329
330 const std::string
331 key(int64_t fieldindex) const override;
332
333 bool
334 haskey(const std::string& key) const override;
335
336 const std::vector<std::string>
337 keys() const override;
338
339 bool
340 istuple() const override;
341
342 // operations
343 const std::string
344 validityerror(const std::string& path) const override;
345
349 const ContentPtr
350 shallow_simplify() const override;
351
352 const ContentPtr
353 num(int64_t axis, int64_t depth) const override;
354
355 const std::pair<Index64, ContentPtr>
356 offsets_and_flattened(int64_t axis, int64_t depth) const override;
357
358 bool
359 mergeable(const ContentPtr& other, bool mergebool) const override;
360
361 bool
362 referentially_equal(const ContentPtr& other) const override;
363
364 const ContentPtr
365 reverse_merge(const ContentPtr& other) const override;
366
367 const ContentPtr
368 mergemany(const ContentPtrVec& others) const override;
369
370 const SliceItemPtr
371 asslice() const override;
372
373 const ContentPtr
374 fillna(const ContentPtr& value) const override;
375
376 const ContentPtr
377 rpad(int64_t target, int64_t axis, int64_t depth) const override;
378
379 const ContentPtr
380 rpad_and_clip(int64_t target,
381 int64_t axis,
382 int64_t depth) const override;
383
384 const ContentPtr
385 reduce_next(const Reducer& reducer,
386 int64_t negaxis,
387 const Index64& starts,
388 const Index64& shifts,
389 const Index64& parents,
390 int64_t outlength,
391 bool mask,
392 bool keepdims) const override;
393
394 const ContentPtr
395 sort_next(int64_t negaxis,
396 const Index64& starts,
397 const Index64& parents,
398 int64_t outlength,
399 bool ascending,
400 bool stable) const override;
401
402 const ContentPtr
403 argsort_next(int64_t negaxis,
404 const Index64& starts,
405 const Index64& shifts,
406 const Index64& parents,
407 int64_t outlength,
408 bool ascending,
409 bool stable) const override;
410
411 const ContentPtr
412 localindex(int64_t axis, int64_t depth) const override;
413
414 const ContentPtr
415 combinations(int64_t n,
416 bool replacement,
417 const util::RecordLookupPtr& recordlookup,
418 const util::Parameters& parameters,
419 int64_t axis,
420 int64_t depth) const override;
421
422 const ContentPtr
424 const Slice& tail,
425 const Index64& advanced) const override;
426
427 const ContentPtr
429 const Slice& tail,
430 const Index64& advanced) const override;
431
432 const ContentPtr
434 const Slice& tail,
435 const Index64& advanced) const override;
436
437 const ContentPtr
439 const Slice& tail,
440 const Index64& advanced) const override;
441
442 const ContentPtr
443 getitem_next_jagged(const Index64& slicestarts,
444 const Index64& slicestops,
445 const SliceArray64& slicecontent,
446 const Slice& tail) const override;
447
448 const ContentPtr
449 getitem_next_jagged(const Index64& slicestarts,
450 const Index64& slicestops,
451 const SliceMissing64& slicecontent,
452 const Slice& tail) const override;
453
454 const ContentPtr
455 getitem_next_jagged(const Index64& slicestarts,
456 const Index64& slicestops,
457 const SliceJagged64& slicecontent,
458 const Slice& tail) const override;
459
460 const ContentPtr
461 copy_to(kernel::lib ptr_lib) const override;
462
463 const ContentPtr
464 numbers_to_type(const std::string& name) const override;
465
467 bool
468 is_unique() const override;
469
471 const ContentPtr
472 unique() const override;
473
475 bool
476 is_subrange_equal(const Index64& start, const Index64& stop) const override;
477
478 private:
480 const IndexU8 mask_;
482 const ContentPtr content_;
484 const bool valid_when_;
486 const int64_t length_;
488 const bool lsb_order_;
489 };
490
491}
492
493#endif // AWKWARD_BITMASKEDARRAY_H_
Represents potentially missing data by overlaying a bit mask over its content.
Definition: BitMaskedArray.h:127
int64_t purelist_depth() const override
The list-depth of this array, not counting any contained within a RecordArray.
const ContentPtr argsort_next(int64_t negaxis, const Index64 &starts, const Index64 &shifts, const Index64 &parents, int64_t outlength, bool ascending, bool stable) const override
This array sorted indices.
const Index8 bytemask() const
Expands the mask to a byte-valued mask with a fixed interpretation: missing values are 1 and valid va...
bool lsb_order() const
If true, the bits in each byte of the mask are taken to be in Least Significant Bit (LSB) order; if f...
void caches(std::vector< ArrayCachePtr > &out) const override
Accumulates all the unique #ArrayCache objects from nested #VirtualArray nodes. (Uniqueness is determ...
const ContentPtr getitem_fields(const std::vector< std::string > &keys) const override
This array with the first nested RecordArray replaced by a RecordArray of a given subset of keys.
const ContentPtr getitem_next_jagged(const Index64 &slicestarts, const Index64 &slicestops, const SliceArray64 &slicecontent, const Slice &tail) const override
Internal function that propagates a jagged array (array with irregular-length dimensions) slice throu...
const ContentPtr getitem_at(int64_t at) const override
Returns the element at a given position in the array, handling negative indexing and bounds-checking ...
void setidentities() override
Assign a surrogate index of Identities to this array (in-place).
const ContentPtr getitem_field(const std::string &key, const Slice &only_fields) const override
const ContentPtr getitem_next(const SliceArray64 &array, const Slice &tail, const Index64 &advanced) const override
Internal function that propagates a generic getitem request through one axis (including advanced inde...
const ContentPtr unique() const override
Returns an array where all components are unique.
const ContentPtr sort_next(int64_t negaxis, const Index64 &starts, const Index64 &parents, int64_t outlength, bool ascending, bool stable) const override
This array sorted.
const std::shared_ptr< IndexedArrayOf< int64_t, true > > toIndexedOptionArray64() const
Converts this array into an IndexedOptionArray with the same missing values.
const ContentPtr reverse_merge(const ContentPtr &other) const override
Merges a single other with this array in reverse order: other first, this last.
const ContentPtr project() const
Return an array with the same type as content with None values removed.
void tojson_part(ToJson &builder, bool include_beginendlist) const override
Internal function to produce a JSON representation one node at a time.
const ContentPtr getitem_fields(const std::vector< std::string > &keys, const Slice &only_fields) const override
int64_t length() const override
The number of elements in the array.
const ContentPtr deep_copy(bool copyarrays, bool copyindexes, bool copyidentities) const override
Copies this node and all nodes hierarchically nested within it, optionally copying the associated arr...
const std::string validityerror(const std::string &path) const override
Returns an error message if this array is invalid; otherwise, returns an empty string.
const ContentPtr getitem_next(const SliceAt &at, const Slice &tail, const Index64 &advanced) const override
Internal function that propagates a generic getitem request through one axis (including advanced inde...
bool istuple() const override
Returns true if the outermost RecordArray is a tuple.
const ContentPtr getitem_nothing() const override
Internal function to get an empty slice (with the correct type).
kernel::lib kernels() const override
Returns the kernel library enum for all nested ptr_lib within the array's tree structure....
const ContentPtr carry(const Index64 &carry, bool allow_lazy) const override
Returns an array of the same type with elements filtered, rearranged, and possibly duplicated by the ...
const ContentPtr combinations(int64_t n, bool replacement, const util::RecordLookupPtr &recordlookup, const util::Parameters &parameters, int64_t axis, int64_t depth) const override
Tuples or records of all n-tuple combinations of list items at some axis depth.
void setidentities(const IdentitiesPtr &identities) override
Assign a specified set of Identities to this array (in-place).
const ContentPtr localindex(int64_t axis, int64_t depth) const override
A (possibly nested) array of integers indicating the positions of elements within each nested list.
const ContentPtr getitem_range_nowrap(int64_t start, int64_t stop) const override
Subinterval of this array, without handling negative indexing or bounds-checking.
bool valid_when() const
Interpretation of the boolean bytes in mask as None or valid values from content. Only boolean bytes ...
const std::pair< Index64, ContentPtr > offsets_and_flattened(int64_t axis, int64_t depth) const override
Returns (a) an offsets Index and (b) a flattened version of the array at some axis depth.
int64_t fieldindex(const std::string &key) const override
The position of a tuple or record key name if this array contains a RecordArray.
void check_for_iteration() const override
Performs up-front validity checks on an array so that they don't have to be checked in getitem_at_now...
const std::vector< std::string > keys() const override
A list of RecordArray keys or an empty list if this array does not contain a RecordArray.
const ContentPtr getitem_field(const std::string &key) const override
This array with the first nested RecordArray replaced by the field at key.
const std::pair< bool, int64_t > branch_depth() const override
Returns (a) whether the list-depth of this array "branches," or differs when followed through differe...
const ContentPtr rpad(int64_t target, int64_t axis, int64_t depth) const override
If axis = 0, returns a view of this array padded on the right with None values to have a minimum leng...
const ContentPtr getitem_next(const SliceItemPtr &head, const Slice &tail, const Index64 &advanced) const override
Internal function that propagates a generic getitem request through one axis (including advanced inde...
const ContentPtr getitem_at_nowrap(int64_t at) const override
Returns the element at a given position in the array, without handling negative indexing or bounds-ch...
void nbytes_part(std::map< size_t, int64_t > &largest) const override
Internal function used to calculate nbytes.
const ContentPtr mergemany(const ContentPtrVec &others) const override
Returns an array with this and the others concatenated (in order, this first, others last).
const std::string key(int64_t fieldindex) const override
The record name associated with a given field index or the tuple index as a string (e....
const ContentPtr shallow_copy() const override
Copies this node without copying any nodes hierarchically nested within it or any array/index/identit...
const std::pair< int64_t, int64_t > minmax_depth() const override
Returns (a) the minimum list-depth and (b) the maximum list-depth of the array, which can differ if t...
bool is_unique() const override
Returns 'true' if all components of the array are unique.
const FormPtr form(bool materialize) const override
Low-level Form describing all the features of this array except the actual data buffers (Index,...
const SliceItemPtr asslice() const override
Converts this array into a SliceItem that can be used in getitem.
const std::string tostring_part(const std::string &indent, const std::string &pre, const std::string &post) const override
Internal function to build an output string for tostring.
const ContentPtr rpad_and_clip(int64_t target, int64_t axis, int64_t depth) const override
If axis = 0, returns a view of this array padded on the right.
const ContentPtr getitem_next(const SliceJagged64 &jagged, const Slice &tail, const Index64 &advanced) const override
Internal function that propagates a generic getitem request through one axis (including advanced inde...
const ContentPtr getitem_range(int64_t start, int64_t stop) const override
Subinterval of this array, handling negative indexing and bounds-checking like Python.
bool is_subrange_equal(const Index64 &start, const Index64 &stop) const override
Returns 'true' if subranges are equal.
const ContentPtr fillna(const ContentPtr &value) const override
Returns this array with None values replaced by a given value.
int64_t numfields() const override
The number of fields in the first nested tuple or records or -1 if this array does not contain a Reco...
bool referentially_equal(const ContentPtr &other) const override
Returns true if this array has all the same buffers and parameters as other; false otherwise.
const TypePtr type(const util::TypeStrs &typestrs) const override
High-level Type describing this array.
const IndexU8 mask() const
Mask in which each bit represents a missing value (None) or a valid value (from content).
const ContentPtr num(int64_t axis, int64_t depth) const override
The length of this array (as a NumpyArray scalar) if axis = 0 or the lengths of subarrays (as an arra...
const ContentPtr project(const Index8 &mask) const
Performs a set-union of a given mask with the missing values and calls project.
const ContentPtr simplify_optiontype() const
If the content also has OptionType, combine the mask with the content's indicator of missing values; ...
bool mergeable(const ContentPtr &other, bool mergebool) const override
Returns true if this array can be merged with the other; false otherwise.
bool haskey(const std::string &key) const override
Returns true if the array contains a RecordArray with the specified key; false otherwise.
const std::shared_ptr< ByteMaskedArray > toByteMaskedArray() const
Converts this array into a ByteMaskedArray.
const ContentPtr getitem_next_jagged(const Index64 &slicestarts, const Index64 &slicestops, const SliceJagged64 &slicecontent, const Slice &tail) const override
Internal function that propagates a jagged array (array with irregular-length dimensions) slice throu...
const ContentPtr getitem_next_jagged(const Index64 &slicestarts, const Index64 &slicestops, const SliceMissing64 &slicecontent, const Slice &tail) const override
Internal function that propagates a jagged array (array with irregular-length dimensions) slice throu...
const ContentPtr getitem_next(const SliceRange &range, const Slice &tail, const Index64 &advanced) const override
Internal function that propagates a generic getitem request through one axis (including advanced inde...
const ContentPtr shallow_simplify() const override
Returns an equivalent array simplified at one level only using simplify_optiontype if an option-type ...
const ContentPtr content() const
Data to be masked; mask[i] corresponds to content[i*8] through content[i*8 + 7] (inclusive) for all i...
const ContentPtr reduce_next(const Reducer &reducer, int64_t negaxis, const Index64 &starts, const Index64 &shifts, const Index64 &parents, int64_t outlength, bool mask, bool keepdims) const override
This array with one axis removed by applying a Reducer (e.g. "sum", "max", "any", "all).
const ContentPtr numbers_to_type(const std::string &name) const override
Change the leaf types to 'totype'.
const std::string classname() const override
User-friendly name of this class: "BitMaskedArray".
const ContentPtr copy_to(kernel::lib ptr_lib) const override
Recursively copies components of the array from main memory to a GPU (if ptr_lib == kernel::lib::cuda...
BitMaskedArray(const IdentitiesPtr &identities, const util::Parameters &parameters, const IndexU8 &mask, const ContentPtr &content, bool valid_when, int64_t length, bool lsb_order)
Creates an BitMaskedArray from a full set of parameters.
Form describing BitMaskedArray.
Definition: BitMaskedArray.h:24
int64_t purelist_depth() const override
The list-depth of this array, not counting any contained within a RecordForm.
const FormPtr with_form_key(const FormKey &form_key) const override
Copies this node, adding or replacing a form_key.
bool lsb_order() const
bool equal(const FormPtr &other, bool check_identities, bool check_parameters, bool check_form_key, bool compatibility_check) const override
Returns true if this Form is equal to the other Form; false otherwise.
const FormPtr simplify_optiontype() const
virtual const FormPtr getitem_range() const override
Returns the Form that would result from a range-slice.
bool istuple() const override
Returns true if the outermost RecordArray is a tuple.
const FormPtr shallow_copy() const override
Copies this node without copying any nodes hierarchically nested within it.
void tojson_part(ToJson &builder, bool verbose) const override
Internal function to produce a JSON representation one node at a time.
const FormPtr getitem_fields(const std::vector< std::string > &keys) const override
Returns the Form that would result from a fields-slice.
bool valid_when() const
int64_t fieldindex(const std::string &key) const override
The position of a tuple or record key name if this array contains a RecordForm.
const std::vector< std::string > keys() const override
A list of RecordArray keys or an empty list if this array does not contain a RecordArray.
Index::Form mask() const
const std::pair< bool, int64_t > branch_depth() const override
Returns (a) whether the list-depth of this array "branches," or differs when followed through differe...
const std::string key(int64_t fieldindex) const override
The record name associated with a given field index or the tuple index as a string (e....
const std::pair< int64_t, int64_t > minmax_depth() const override
Returns (a) the minimum list-depth and (b) the maximum list-depth of the array, which can differ if t...
const FormPtr content() const
const FormPtr getitem_field(const std::string &key) const override
Returns the Form that would result from a field-slice.
int64_t numfields() const override
The number of fields in the first nested tuple or records or -1 if this array does not contain a Reco...
const TypePtr type(const util::TypeStrs &typestrs) const override
High-level Type describing this Form.
const std::string purelist_parameter(const std::string &key) const override
The parameter associated with key at the first level that has a non-null value, descending only as de...
bool haskey(const std::string &key) const override
Returns true if the array contains a RecordForm with the specified key; false otherwise.
bool purelist_isregular() const override
Returns true if all nested lists down to the first RecordForm are RegularForm nodes; false otherwise.
BitMaskedForm(bool has_identities, const util::Parameters &parameters, const FormKey &form_key, Index::Form mask, const FormPtr &content, bool valid_when, bool lsb_order)
Creates a BitMaskedForm. See BitMaskedArray for documentation.
bool dimension_optiontype() const override
Returns true if this dimension has option-type; false otherwise.
Represents potentially missing data by overlaying a byte mask over its content.
Definition: ByteMaskedArray.h:117
Abstract superclass of all array node types (flat hierarchy). Any Content can be nested within any ot...
Definition: Content.h:276
Abstract superclass of all array node forms, which expresses the nesting structure without any large ...
Definition: Content.h:39
A contiguous, one-dimensional array of integers used to represent data structures,...
Definition: Index.h:82
Form
Integer type of an Index, used by ListForm, IndexedForm, etc.
Definition: Index.h:34
Filters, rearranges, and/or duplicates items in its content through an index, which has the same effe...
Definition: IndexedArray.h:204
Abstract class for all reducer algorithms.
Definition: Reducer.h:20
Represents an array of integers in a slice (possibly converted from an array of booleans).
Definition: Slice.h:229
Represents an integer in a tuple of slices passed to __getitem__ in Python.
Definition: Slice.h:58
Represents an array of nested lists, where the content may be SliceArrayOf, SliceMissingOf,...
Definition: Slice.h:515
Represents a SliceArrayOf, SliceMissingOf, or SliceJaggedOf with missing values: None (no equivalent ...
Definition: Slice.h:435
Represents a Python slice object (usual syntax: array[start:stop:step]).
Definition: Slice.h:93
A sequence of SliceItem objects representing a tuple passed to Python's __getitem__.
Definition: Slice.h:585
Definition: json.h:21
#define LIBAWKWARD_EXPORT_SYMBOL
Definition: common.h:45
lib
Definition: kernel-dispatch.h:20
std::map< std::string, std::string > Parameters
Definition: util.h:165
std::shared_ptr< RecordLookup > RecordLookupPtr
Definition: util.h:130
std::map< std::string, std::string > TypeStrs
Definition: util.h:215
Definition: BitMaskedArray.h:15
std::shared_ptr< std::string > FormKey
Definition: Content.h:19
std::vector< std::shared_ptr< Content > > ContentPtrVec
Definition: Content.h:16
std::shared_ptr< SliceItem > SliceItemPtr
Definition: Slice.h:15
std::shared_ptr< Content > ContentPtr
Definition: Content.h:15
std::shared_ptr< Form > FormPtr
Definition: Content.h:18
std::shared_ptr< Type > TypePtr
Definition: Content.h:23
std::shared_ptr< Identities > IdentitiesPtr
Definition: Identities.h:16