Struct ParquetOpener 

pub(crate) struct ParquetOpener {

    pub partition_index: usize,
    pub projection: Arc<[usize]>,
    pub batch_size: usize,
    pub limit: Option<usize>,
    pub predicate: Option<Arc<dyn PhysicalExpr>>,
    pub logical_file_schema: SchemaRef,
    pub partition_fields: Vec<FieldRef>,
    pub metadata_size_hint: Option<usize>,
    pub metrics: ExecutionPlanMetricsSet,
    pub parquet_file_reader_factory: Arc<dyn ParquetFileReaderFactory>,
    pub pushdown_filters: bool,
    pub reorder_filters: bool,
    pub enable_page_index: bool,
    pub enable_bloom_filter: bool,
    pub schema_adapter_factory: Arc<dyn SchemaAdapterFactory>,
    pub enable_row_group_stats_pruning: bool,
    pub coerce_int96: Option<TimeUnit>,
    pub file_decryption_properties: Option<Arc<FileDecryptionProperties>>,
    pub(crate) expr_adapter_factory: Option<Arc<dyn PhysicalExprAdapterFactory>>,
    pub encryption_factory: Option<(Arc<dyn EncryptionFactory>, EncryptionFactoryOptions)>,
    pub max_predicate_cache_size: Option<usize>,
}

Implements FileOpener for a parquet file

Fields

partition_index: usize

Execution partition index

projection: Arc<[usize]>

Column indexes in table_schema needed by the query

batch_size: usize

Target number of rows in each output RecordBatch

limit: Option<usize>

Optional limit on the number of rows to read

predicate: Option<Arc<dyn PhysicalExpr>>

Optional predicate to apply during the scan

logical_file_schema: SchemaRef

Schema of the output table without partition columns. This is the schema we coerce the physical file schema into.

partition_fields: Vec<FieldRef>

Partition columns

metadata_size_hint: Option<usize>

Optional hint for how large the initial request to read parquet metadata should be

metrics: ExecutionPlanMetricsSet

Metrics for reporting

parquet_file_reader_factory: Arc<dyn ParquetFileReaderFactory>

Factory for instantiating parquet reader

pushdown_filters: bool

Should the filters be evaluated during the parquet scan using DataFusionArrowPredicate?

reorder_filters: bool

Should the filters be reordered to optimize the scan?

enable_page_index: bool

Should the page index be read from parquet files, if present, to skip data pages

enable_bloom_filter: bool

Should the bloom filter be read from parquet, if present, to skip row groups

schema_adapter_factory: Arc<dyn SchemaAdapterFactory>

Schema adapter factory

enable_row_group_stats_pruning: bool

Should row group pruning be applied

coerce_int96: Option<TimeUnit>

Coerce INT96 timestamps to specific TimeUnit

file_decryption_properties: Option<Arc<FileDecryptionProperties>>

Optional parquet FileDecryptionProperties

expr_adapter_factory: Option<Arc<dyn PhysicalExprAdapterFactory>>

Rewrite expressions in the context of the file schema

encryption_factory: Option<(Arc<dyn EncryptionFactory>, EncryptionFactoryOptions)>

Optional factory to create file decryption properties dynamically

max_predicate_cache_size: Option<usize>

Maximum size of the predicate cache, in bytes. If none, uses the arrow-rs default.

Implementations

impl ParquetOpener

fn get_encryption_context(&self) -> EncryptionContext

Trait Implementations

impl FileOpener for ParquetOpener

fn open(&self, partitioned_file: PartitionedFile) -> Result<FileOpenFuture>

Asynchronously open the specified file and return a stream of [RecordBatch]