Index

A

add_note() (PipelineConfigError method), [1] apply() (Dag method), [1] (LinearDag method), [1]

apply_dag() (Pipeline method), [1] ApplySlidingWindows (class in gfw.common.beam.transforms), [1]

B

bigquery_helper_factory (DagFactory property), [1] (LinearDagFactory property) BigQueryHelper (class in gfw.common.bigquery), [1] binary_search_first_ge() (in module gfw.common.iterables), [1]

build_dag() (DagFactory method), [1] (LinearDagFactory method), [1] build_pipeline() (PipelineFactory method), [1] builtin_options() (CLI class method), [1]

C

caveats (TableDescription attribute), [1] chunked_it() (in module gfw.common.iterables), [1] clear() (DeepChainMap method) CLI (class in gfw.common.cli), [1] client (BigQueryHelper property), [1] cloud_options (Pipeline property), [1] clustering_fields (TableConfig attribute), [1] collapse_paragraphs() (in module gfw.common.strings), [1] Command (class in gfw.common.cli), [1] common_parser (CLI property), [1] compare_items() (in module gfw.common.diff), [1]

copy() (DeepChainMap method) copy_dict_without() (in module gfw.common.dictionaries), [1] copy_with() (Command method), [1] (ParametrizedCommand method) core (LinearDagFactory property), [1] counter() (in module gfw.common.decorators), [1] create_external_table() (BigQueryHelper method), [1] create_label() (GroupBy class method), [1] create_table() (BigQueryHelper method), [1] create_table_hook() (in module gfw.common.beam.pipeline.hooks), [1] create_view() (BigQueryHelper method), [1] create_view_hook() (in module gfw.common.beam.pipeline.hooks), [1]

D

Dag (class in gfw.common.beam.pipeline), [1] DagFactory (class in gfw.common.beam.pipeline), [1] date_range (PipelineConfig attribute), [1] datetime_from_date() (in module gfw.common.datetime), [1] datetime_from_isoformat() (in module gfw.common.datetime), [1] datetime_from_string() (in module gfw.common.datetime), [1] datetime_from_timestamp() (in module gfw.common.datetime), [1] datetime_to_timestamp() (Query class method), [1] DeepChainMap (class in gfw.common.collections), [1] DEFAULT_JINJA_FOLDER (Query attribute), [1] default_options() (Pipeline static method), [1] defaults() (Command method), [1] (ParametrizedCommand method) delete_events_hook() (in module gfw.common.beam.pipeline.hooks), [1] delete_query() (TableConfig method), [1]

description (Command property), [1] (ParametrizedCommand property), [1] (TableConfig attribute), [1] dest (Option property), [1] diff_lines() (in module gfw.common.diff), [1] display_data() (ApplySlidingWindows method) (FakeReadFromPubSub method) (FakeWriteToBigQuery method) (GroupBy method) (ReadAndDecodeFromPubSub method) (ReadFromBigQuery method) (ReadFromJson method) (ReadMatchingAvroFiles method) (SampleAndLogElements method) (WriteToBigQueryWrapper method) (WriteToJson method)

E

end_date (PipelineConfig property), [1] end_session() (BigQueryHelper method), [1] EnvironmentLoader (class in gfw.common.jinja2), [1] equal_to() (in module gfw.common.beam.testing), [1] execute() (CLI method), [1] expand() (ApplySlidingWindows method), [1] (FakeReadFromPubSub method), [1] (FakeWriteToBigQuery method), [1] (GroupBy method), [1] (ReadAndDecodeFromPubSub method), [1] (ReadFromBigQuery method), [1] (ReadFromJson method), [1] (ReadMatchingAvroFiles method), [1] (SampleAndLogElements method), [1] (WriteToBigQueryWrapper method), [1] (WriteToJson method), [1]

F

FakeReadFromPubSub (class in gfw.common.beam.transforms), [1] FakeWriteToBigQuery (class in gfw.common.beam.transforms), [1] filter_none_values() (in module gfw.common.dictionaries), [1] float_to_beam_timestamp() (WriteToBigQueryWrapper static method), [1] format() (Query static method), [1]

format_jinja2() (BigQueryHelper static method), [1] from_namespace() (PipelineConfig class method), [1] from_package() (EnvironmentLoader method), [1] from_query() (ReadFromBigQuery class method), [1] fromkeys() (DeepChainMap class method)

G

get() (DeepChainMap method) get_client_factory() (BigQueryHelper class method), [1] (ReadAndDecodeFromPubSub class method), [1] (ReadFromBigQuery class method), [1] (WriteToBigQueryWrapper class method), [1] get_dict_table_schema() (FakeWriteToBigQuery static method) get_select_fields() (Query method), [1] get_table_schema_from_string() (FakeWriteToBigQuery static method) get_type_hints() (ApplySlidingWindows method) (FakeReadFromPubSub method) (FakeWriteToBigQuery method) (GroupBy method) (ReadAndDecodeFromPubSub method) (ReadFromBigQuery method) (ReadFromJson method) (ReadMatchingAvroFiles method) (SampleAndLogElements method) (WriteToBigQueryWrapper method) (WriteToJson method) get_windowing() (ApplySlidingWindows method) (FakeReadFromPubSub method) (FakeWriteToBigQuery method) (GroupBy method) (ReadAndDecodeFromPubSub method) (ReadFromBigQuery method) (ReadFromJson method) (ReadMatchingAvroFiles method) (SampleAndLogElements method) (WriteToBigQueryWrapper method) (WriteToJson method) gfw.common.beam module gfw.common.beam.pipeline module gfw.common.beam.pipeline.hooks module

gfw.common.beam.testing module gfw.common.beam.transforms module gfw.common.bigquery module gfw.common.cli module gfw.common.collections module gfw.common.config module gfw.common.datetime module gfw.common.decorators module gfw.common.dictionaries module gfw.common.diff module gfw.common.io module gfw.common.iterables module gfw.common.jinja2 module gfw.common.logging module gfw.common.query module gfw.common.serialization module gfw.common.sorting module gfw.common.strings module GroupBy (class in gfw.common.beam.transforms), [1]

H

header (Command property), [1] (ParametrizedCommand property)

I

is_path_in_range() (ReadMatchingAvroFiles method), [1] is_streaming (Pipeline property), [1]

items() (DeepChainMap method) iter_as_dicts() (QueryResult method), [1]

J

jinja_env (PipelineConfig property), [1] (Query property), [1]

jinja_folder (PipelineConfig attribute), [1] json_load() (in module gfw.common.io), [1] json_save() (in module gfw.common.io), [1]

K

keys() (DeepChainMap method)

L

LinearDag (class in gfw.common.beam.pipeline), [1] LinearDagFactory (class in gfw.common.beam.pipeline), [1]

load_from_json() (BigQueryHelper method), [1] LoggerConfig (class in gfw.common.logging), [1]

M

main_parser (CLI property), [1] mock_bq_clients (PipelineConfig attribute), [1] mocked() (BigQueryHelper class method), [1] module gfw.common.beam gfw.common.beam.pipeline gfw.common.beam.pipeline.hooks gfw.common.beam.testing gfw.common.beam.transforms gfw.common.bigquery gfw.common.cli gfw.common.collections gfw.common.config gfw.common.datetime gfw.common.decorators gfw.common.dictionaries gfw.common.diff gfw.common.io gfw.common.iterables gfw.common.jinja2 gfw.common.logging gfw.common.query gfw.common.serialization gfw.common.sorting gfw.common.strings

N

name (Command property), [1] (ParametrizedCommand property), [1] (PipelineConfig attribute), [1]

new_child() (DeepChainMap method)

O

Option (class in gfw.common.cli), [1] options (Command property), [1] (ParametrizedCommand property), [1]

output_paths (LinearDag attribute), [1] (LinearDag property), [1] output_type (Query property), [1]

P

ParametrizedCommand (class in gfw.common.cli), [1] parents (DeepChainMap property) parsed_args (Pipeline attribute), [1] (Pipeline property), [1] parsed_date_range (PipelineConfig property), [1] partition_field (TableConfig attribute), [1] partition_type (TableConfig attribute), [1] Pipeline (class in gfw.common.beam.pipeline), [1] pipeline (Pipeline attribute), [1] (Pipeline property), [1]

pipeline_options (Pipeline attribute), [1] (Pipeline property), [1] PipelineConfig (class in gfw.common.config), [1] PipelineConfigError, [1] PipelineFactory (class in gfw.common.beam.pipeline), [1] pop() (DeepChainMap method) popitem() (DeepChainMap method) post_hooks (PipelineConfig property), [1] pre_hooks (PipelineConfig property), [1]

Q

Query (class in gfw.common.query), [1]

query_job (QueryResult attribute), [1] QueryResult (class in gfw.common.bigquery), [1]

R

read_from_bigquery_factory (DagFactory property), [1] (LinearDagFactory property) ReadAndDecodeFromPubSub (class in gfw.common.beam.transforms), [1] ReadFromBigQuery (class in gfw.common.beam.transforms), [1] ReadFromJson (class in gfw.common.beam.transforms), [1] ReadMatchingAvroFiles (class in gfw.common.beam.transforms), [1] relevant_params (TableDescription attribute), [1] render() (Query method), [1] (TableDescription method), [1]

render_diff_panel() (in module gfw.common.diff), [1] repo_name (TableDescription attribute), [1] resolve_write_method() (WriteToBigQueryWrapper static method), [1] row_iterator (QueryResult attribute), [1] run() (Command method), [1], [2], [3] (ParametrizedCommand method), [1] (Pipeline method), [1] run_query() (BigQueryHelper method), [1]

S

SampleAndLogElements (class in gfw.common.beam.transforms), [1] schema (TableConfig property), [1] (WriteToBigQueryWrapper property), [1] schema_file (TableConfig attribute), [1] session_id (QueryResult property), [1] setdefault() (DeepChainMap method) setup() (LoggerConfig method), [1] side_inputs (LinearDagFactory property), [1]

sinks (LinearDagFactory property), [1] sort_dicts() (in module gfw.common.sorting), [1] sources (LinearDagFactory property), [1] sql_strings() (Query static method), [1] start_date (PipelineConfig property), [1] subscription (ReadAndDecodeFromPubSub property), [1] subtitle (TableDescription attribute), [1] summary (TableDescription attribute), [1]

T

table_id (TableConfig attribute), [1] table_schema_to_dict() (FakeWriteToBigQuery static method) TableConfig (class in gfw.common.bigquery), [1] TableDescription (class in gfw.common.bigquery), [1] template_filename (Query property), [1] template_vars (Query property), [1] timestamp_fields (WriteToBigQueryWrapper property), [1] timing() (in module gfw.common.decorators), [1]

title (CLI property), [1] (TableDescription attribute), [1] to_bigquery_params() (TableConfig method), [1] to_dict() (DeepChainMap method), [1] (PipelineConfig method), [1] to_json() (in module gfw.common.serialization), [1] tolist() (QueryResult method), [1] top_level_package (PipelineConfig property), [1] (Query property), [1]

U

unknown_parsed_args (PipelineConfig attribute), [1]

unknown_unparsed_args (PipelineConfig attribute), [1] update() (DeepChainMap method)

V

values() (DeepChainMap method) version (PipelineConfig attribute), [1] (TableDescription attribute), [1]

view_id (TableConfig property), [1] view_query() (TableConfig method), [1] view_suffix (TableConfig attribute), [1]

W

with_env() (Query method), [1] with_input_types() (ApplySlidingWindows method) (FakeReadFromPubSub method) (FakeWriteToBigQuery method) (GroupBy method) (ReadAndDecodeFromPubSub method) (ReadFromBigQuery method) (ReadFromJson method) (ReadMatchingAvroFiles method) (SampleAndLogElements method) (WriteToBigQueryWrapper method) (WriteToJson method) with_output_types() (ApplySlidingWindows method) (FakeReadFromPubSub method) (FakeWriteToBigQuery method) (GroupBy method) (ReadAndDecodeFromPubSub method) (ReadFromBigQuery method) (ReadFromJson method) (ReadMatchingAvroFiles method) (SampleAndLogElements method) (WriteToBigQueryWrapper method) (WriteToJson method)

with_resource_hints() (ApplySlidingWindows method) (FakeReadFromPubSub method) (FakeWriteToBigQuery method) (GroupBy method) (ReadAndDecodeFromPubSub method) (ReadFromBigQuery method) (ReadFromJson method) (ReadMatchingAvroFiles method) (SampleAndLogElements method) (WriteToBigQueryWrapper method) (WriteToJson method) with_traceback() (PipelineConfigError method), [1] write_to_bigquery_factory (DagFactory property), [1] (LinearDagFactory property) WriteToBigQueryWrapper (class in gfw.common.beam.transforms), [1] WriteToJson (class in gfw.common.beam.transforms), [1]

Y

yaml_load() (in module gfw.common.io), [1]

yaml_save() (in module gfw.common.io), [1]