Index

_

__fspath__() (xffl.custom.types._PathLike method) __post_init__() (xffl.custom.config.DatasetInfo method) (xffl.custom.config.ModelInfo method) (xffl.custom.config.XFFLConfig method) __str__() (xffl.custom.types._PathLike method) (xffl.distributed.distributed_state.DistributedState method) _add_common_project_options() (in module xffl.cli.parser) _all_reduce() (in module xffl.distributed.aggregation) _apply_filters() (in module xffl.learning.data) _apply_subsampling() (in module xffl.learning.data) _check_and_create_dir() (in module xffl.cli.config) _configure_facility() (in module xffl.cli.config) _dump_cwl() (in module xffl.cli.config) _fedopt_setup() (in module xffl.learning.processing) _fedopt_step() (in module xffl.learning.processing) _get_communicating_processes() (xffl.distributed.distributed_state.DistributedState method) _get_current_device() (in module xffl.distributed.distributed) _get_default_nodelist() (in module xffl.cli.parser) _get_default_ppn() (in module xffl.cli.parser) _get_facilitator_path() (in module xffl.cli.exec) _get_federated_group() (in module xffl.distributed.aggregation) _get_global_fsdp_mesh() (xffl.distributed.distributed_state.DistributedState method)

_get_init_device() (in module xffl.distributed.distributed) _get_int_from_env() (in module xffl.distributed.distributed) _get_model_info() (in module xffl.cli.config) _get_processing_function() (in module xffl.learning.processing) _get_src_rank() (in module xffl.distributed.aggregation) _get_stream_context() (in module xffl.distributed.aggregation) _get_training_info() (in module xffl.cli.config) _import_from_path() (in module xffl.cli.exec) _init_distributed_process_group() (in module xffl.distributed.distributed) _is_broadcast_necessary() (in module xffl.distributed.aggregation) _move_opt_to() (in module xffl.learning.processing) _partial_hsdp_setup() (xffl.distributed.distributed_state.DistributedState method) _partial_hsdp_setup_manual() (xffl.distributed.distributed_state.DistributedState method) _PathLike (class in xffl.custom.types) _set_asymmetric_federated_scaling() (xffl.distributed.distributed_state.DistributedState method) _set_global_hsdp_mesh() (xffl.distributed.distributed_state.DistributedState method) _set_rank_role() (xffl.distributed.distributed_state.DistributedState method) _set_symmetric_federated_scaling() (xffl.distributed.distributed_state.DistributedState method) _setup_amp() (in module xffl.learning.processing) _setup_env() (in module xffl.cli.exec) _setup_execution_env() (in module xffl.cli.exec) _setup_streams() (in module xffl.distributed.aggregation) _write_output_files() (in module xffl.cli.config)

A

activation_checkpointing (xffl.custom.config.ModelInfo attribute) add_deployment() (xffl.workflow.templates.streamflow.StreamFlowFile method) add_inputs() (xffl.workflow.templates.cwl.AggregateStep method) (xffl.workflow.templates.cwl.CWLConfig method) (xffl.workflow.templates.cwl.MainWorkflow method) (xffl.workflow.templates.cwl.RoundWorkflow method) (xffl.workflow.templates.cwl.TrainingStep method) (xffl.workflow.templates.cwl.Workflow method) (xffl.workflow.templates.streamflow.StreamFlowFile method)

add_root_step() (xffl.workflow.templates.streamflow.StreamFlowFile method) add_training_step() (xffl.workflow.templates.streamflow.StreamFlowFile method) AggregateStep (class in xffl.workflow.templates.cwl) attention (xffl.custom.config.ModelInfo attribute)

B

backend (xffl.custom.config.XFFLConfig attribute) (xffl.distributed.distributed_state.DistributedState attribute) barrier() (in module xffl.learning.utils) batch_sizes (xffl.custom.config.DatasetInfo attribute) benchmark_aggregation() (in module xffl.distributed.aggregation) BLUE (xffl.custom.formatter.CustomFormatter attribute) BOLD_RED (xffl.custom.formatter.CustomFormatter attribute) broadcast (xffl.distributed.aggregation.Strategy attribute)

bucket_coalesced() (in module xffl.distributed.aggregation) bucket_coalesced_() (in module xffl.distributed.aggregation) bucket_flatten() (in module xffl.distributed.aggregation) bucket_flatten_() (in module xffl.distributed.aggregation) bucket_optimized_coalesced() (in module xffl.distributed.aggregation) bucket_optimized_coalesced_() (in module xffl.distributed.aggregation) bucket_optimized_flatten() (in module xffl.distributed.aggregation) bucket_optimized_flatten_() (in module xffl.distributed.aggregation) build_parser() (in module xffl.cli.parser)

C

check_input() (in module xffl.cli.utils) cleanup_distributed_process_group() (in module xffl.distributed.distributed) collate_fn (xffl.custom.config.DatasetInfo attribute) COMMANDS (in module xffl.cli.cli) config() (in module xffl.cli.config) content (xffl.workflow.config.YamlConfig attribute) create_binding() (xffl.workflow.templates.streamflow.StreamFlowFile static method) create_dataloaders() (in module xffl.learning.data) create_device_mesh() (in module xffl.distributed.distributed_state) create_fsdp_model() (in module xffl.learning.modelling)

create_process_group() (xffl.distributed.distributed_state.DistributedState method) criterion (xffl.custom.config.XFFLConfig attribute) cuda_reset_memory_stats_and_empty_cache() (in module xffl.learning.utils) cuda_streams (xffl.custom.config.XFFLConfig attribute) cuda_sync() (in module xffl.learning.utils) cuda_sync_and_empty_cache() (in module xffl.learning.utils) current_device (xffl.distributed.distributed_state.DistributedState attribute) CustomFormatter (class in xffl.custom.formatter) cwl (xffl.workflow.templates.cwl.Workflow attribute) CWLConfig (class in xffl.workflow.templates.cwl)

D

dataset_info (xffl.custom.config.XFFLConfig attribute) DatasetInfo (class in xffl.custom.config) DATE_FORMAT (xffl.custom.formatter.CustomFormatter attribute) decoder_layer (xffl.custom.config.ModelInfo attribute) DEFAULT_FORMAT (xffl.custom.formatter.CustomFormatter attribute) DEFAULT_xFFL_DIR (in module xffl.utils.constants)

deployments (xffl.workflow.templates.streamflow.StreamFlowFile attribute) device (xffl.custom.config.XFFLConfig attribute) device_type (xffl.distributed.distributed_state.DistributedState attribute) dispatch_command() (in module xffl.cli.cli) distributed_training() (in module xffl.learning.processing) DistributedState (class in xffl.distributed.distributed_state)

E

epochs (xffl.custom.config.XFFLConfig attribute) exclude (xffl.custom.formatter.ExcludeLoggerFilter attribute)

EXCLUDED_LOGGERS (in module xffl.utils.logging) ExcludeLoggerFilter (class in xffl.custom.formatter) exec() (in module xffl.cli.exec)

F

federated (xffl.custom.config.XFFLConfig attribute) federated_batches (xffl.custom.config.XFFLConfig attribute) federated_group (xffl.distributed.distributed_state.DistributedState attribute) federated_local_rank (xffl.distributed.distributed_state.DistributedState attribute) federated_local_size (xffl.distributed.distributed_state.DistributedState attribute) federated_rank (xffl.distributed.distributed_state.DistributedState attribute) federated_world_size (xffl.distributed.distributed_state.DistributedState attribute) federation (xffl.distributed.distributed_state.DistributedState attribute)

FileLike (class in xffl.custom.types) filter() (xffl.custom.formatter.ExcludeLoggerFilter method) filters (xffl.custom.config.DatasetInfo attribute) fmt (xffl.custom.formatter.CustomFormatter attribute) FolderLike (class in xffl.custom.types) format() (xffl.custom.formatter.CustomFormatter method) FORMATS (xffl.custom.formatter.CustomFormatter attribute) fsdp_mesh (xffl.distributed.distributed_state.DistributedState attribute)

G

get_appropriate_sharding_strategy() (in module xffl.distributed.distributed) get_available_position() (xffl.workflow.templates.cwl.TrainingStep method) get_average_distributed_loss() (in module xffl.distributed.aggregation) get_cells_ids() (in module xffl.distributed.networking) get_default_content() (xffl.workflow.config.YamlConfig class method) (xffl.workflow.templates.cwl.CWLConfig class method) (xffl.workflow.templates.streamflow.StreamFlowFile class method) get_default_definition() (xffl.workflow.templates.cwl.AggregateStep class method) (xffl.workflow.templates.cwl.MainWorkflow class method) (xffl.workflow.templates.cwl.RoundWorkflow class method) (xffl.workflow.templates.cwl.TrainingStep class method) (xffl.workflow.templates.cwl.Workflow class method)

get_default_handler() (in module xffl.utils.logging) get_default_nccl_process_group_options() (in module xffl.utils.utils) get_model_size() (in module xffl.learning.utils) get_model_size_in_bits() (in module xffl.learning.utils) get_timeout() (in module xffl.utils.utils) get_training_step() (xffl.workflow.templates.cwl.RoundWorkflow class method) gradient_accumulation (xffl.custom.config.XFFLConfig attribute) gradient_clipping (xffl.custom.config.XFFLConfig attribute) GREY (xffl.custom.formatter.CustomFormatter attribute) group_local_rank (xffl.custom.config.XFFLConfig attribute) group_local_size (xffl.custom.config.XFFLConfig attribute) group_rank (xffl.custom.config.XFFLConfig attribute) group_world_size (xffl.custom.config.XFFLConfig attribute)

H

hsdp (xffl.custom.config.XFFLConfig attribute)

hsdp_mesh (xffl.distributed.distributed_state.DistributedState attribute)

I

init_device (xffl.distributed.distributed_state.DistributedState attribute) is_federated_scaling_setup() (xffl.distributed.distributed_state.DistributedState method) is_fsdp_setup() (xffl.distributed.distributed_state.DistributedState method)

is_hsdp_setup() (xffl.distributed.distributed_state.DistributedState method) is_node_setup() (xffl.distributed.distributed_state.DistributedState method) is_sender (xffl.distributed.distributed_state.DistributedState attribute)

L

layer_by_layer() (in module xffl.distributed.aggregation) layer_by_layer_() (in module xffl.distributed.aggregation) layer_by_layer_optimized() (in module xffl.distributed.aggregation) layer_by_layer_optimized_() (in module xffl.distributed.aggregation) learning_rate (xffl.custom.config.XFFLConfig attribute) load_datasets_from_disk() (in module xffl.learning.data) logger (in module xffl.cli.cli) (in module xffl.cli.config) (in module xffl.cli.exec) (in module xffl.cli.run) (in module xffl.cli.utils) (in module xffl.custom.config) (in module xffl.distributed.aggregation) (in module xffl.distributed.distributed) (in module xffl.distributed.distributed_state) (in module xffl.learning.data) (in module xffl.learning.modelling) (in module xffl.learning.processing) (in module xffl.learning.utils) (in module xffl.utils.utils) (in module xffl.workflow.streamflow) (in module xffl.workflow.templates.cwl)

loglevel (xffl.custom.config.XFFLConfig attribute) lr_scheduler (xffl.custom.config.XFFLConfig attribute)

M

main() (in module xffl.cli.cli) (in module xffl.cli.config) (in module xffl.cli.exec) (in module xffl.cli.run) (in module xffl.workflow.templates.aggregation_application) MainWorkflow (class in xffl.workflow.templates.cwl) mapping (xffl.distributed.aggregation.Strategy attribute) master_addr (xffl.custom.config.XFFLConfig attribute) (xffl.distributed.distributed_state.DistributedState attribute) master_port (xffl.custom.config.XFFLConfig attribute) (xffl.distributed.distributed_state.DistributedState attribute) meta_initialization (xffl.distributed.distributed_state.DistributedState attribute) mixed_precision (xffl.custom.config.ModelInfo attribute) model (xffl.custom.config.ModelInfo attribute) model_info (xffl.custom.config.XFFLConfig attribute) ModelInfo (class in xffl.custom.config) module xffl xffl.cli xffl.cli.cli xffl.cli.config xffl.cli.exec xffl.cli.parser xffl.cli.run xffl.cli.utils xffl.custom xffl.custom.config xffl.custom.formatter xffl.custom.types xffl.distributed xffl.distributed.aggregation xffl.distributed.distributed xffl.distributed.distributed_state xffl.distributed.networking xffl.learning xffl.learning.data xffl.learning.modelling xffl.learning.processing xffl.learning.utils xffl.utils xffl.utils.constants xffl.utils.logging xffl.utils.utils xffl.workflow xffl.workflow.config xffl.workflow.streamflow xffl.workflow.templates xffl.workflow.templates.aggregation_application xffl.workflow.templates.cwl xffl.workflow.templates.streamflow

N

name (xffl.custom.config.DatasetInfo attribute) (xffl.custom.config.ModelInfo attribute) node_local_rank (xffl.distributed.distributed_state.DistributedState attribute)

node_local_size (xffl.distributed.distributed_state.DistributedState attribute) node_rank (xffl.distributed.distributed_state.DistributedState attribute) node_world_size (xffl.distributed.distributed_state.DistributedState attribute)

O

optimizer (xffl.custom.config.XFFLConfig attribute)

output_folder (xffl.custom.config.XFFLConfig attribute) output_model (xffl.custom.config.XFFLConfig attribute)

P

parser (in module xffl.workflow.templates.aggregation_application) path (xffl.custom.config.DatasetInfo attribute) (xffl.custom.config.ModelInfo attribute) (xffl.custom.types._PathLike attribute)

preload() (in module xffl.learning.utils)

R

rank (xffl.custom.config.XFFLConfig attribute) (xffl.distributed.distributed_state.DistributedState attribute) receive_from (xffl.distributed.distributed_state.DistributedState attribute) RED (xffl.custom.formatter.CustomFormatter attribute) reduce_op (xffl.distributed.aggregation.Strategy attribute) replica_group (xffl.distributed.distributed_state.DistributedState attribute) replica_local_rank (xffl.distributed.distributed_state.DistributedState attribute) replica_local_size (xffl.distributed.distributed_state.DistributedState attribute)

replica_rank (xffl.distributed.distributed_state.DistributedState attribute) replica_world_size (xffl.distributed.distributed_state.DistributedState attribute) RESET (xffl.custom.formatter.CustomFormatter attribute) resolve_param() (in module xffl.utils.utils) resolve_path() (in module xffl.cli.utils) RoundWorkflow (class in xffl.workflow.templates.cwl) run() (in module xffl.cli.cli) (in module xffl.cli.run) run_streamflow() (in module xffl.workflow.streamflow)

S

save() (xffl.workflow.config.YamlConfig method) (xffl.workflow.templates.cwl.Workflow method) save_model() (in module xffl.learning.modelling) scale_learning_rate (xffl.custom.config.XFFLConfig attribute) seed (xffl.custom.config.XFFLConfig attribute) seed_dataloader_worker() (in module xffl.learning.utils) set_activation_checkpointing() (in module xffl.learning.utils) set_deterministic_execution() (in module xffl.learning.utils) set_exec_device() (xffl.distributed.distributed_state.DistributedState method) set_federated_scaling() (xffl.distributed.distributed_state.DistributedState method) set_fsdp() (xffl.distributed.distributed_state.DistributedState method) set_global() (xffl.distributed.distributed_state.DistributedState method) set_hsdp() (xffl.distributed.distributed_state.DistributedState method)

set_init_device() (xffl.distributed.distributed_state.DistributedState method) set_node() (xffl.distributed.distributed_state.DistributedState method) set_nondeterministic_execution() (in module xffl.learning.utils) setup_distributed_process_group() (in module xffl.distributed.distributed) setup_logging() (in module xffl.utils.logging) splits (xffl.custom.config.DatasetInfo attribute) src (xffl.distributed.aggregation.Strategy attribute) step_bindings (xffl.workflow.templates.streamflow.StreamFlowFile attribute) Strategy (class in xffl.distributed.aggregation) StreamFlowFile (class in xffl.workflow.templates.streamflow) streams (xffl.distributed.distributed_state.DistributedState attribute) subsampling (xffl.custom.config.DatasetInfo attribute) SUPPORTED_QUEUE_MANAGERS (in module xffl.utils.constants)

T

tokenizer (xffl.custom.config.ModelInfo attribute)

TrainingStep (class in xffl.workflow.templates.cwl)

U

unset_federated_scaling() (xffl.distributed.distributed_state.DistributedState method) unset_hsdp() (xffl.distributed.distributed_state.DistributedState method)

update_merge_step() (xffl.workflow.templates.cwl.RoundWorkflow method) use_contiguous_memory (xffl.distributed.aggregation.Strategy attribute)

V

validation() (in module xffl.learning.processing)

VERSION (in module xffl.utils.constants)

W

wandb_entity (xffl.custom.config.XFFLConfig attribute) wandb_group (xffl.custom.config.XFFLConfig attribute) wandb_mode (xffl.custom.config.XFFLConfig attribute) wandb_name (xffl.custom.config.XFFLConfig attribute) wandb_notes (xffl.custom.config.XFFLConfig attribute) wandb_project (xffl.custom.config.XFFLConfig attribute)

wandb_setup() (in module xffl.learning.utils) wandb_tags (xffl.custom.config.XFFLConfig attribute) workers (xffl.custom.config.DatasetInfo attribute) Workflow (class in xffl.workflow.templates.cwl) world_size (xffl.custom.config.XFFLConfig attribute) (xffl.distributed.distributed_state.DistributedState attribute) wrapping_policy (xffl.custom.config.ModelInfo attribute)

X

xffl module xffl.cli module xffl.cli.cli module xffl.cli.config module xffl.cli.exec module xffl.cli.parser module xffl.cli.run module xffl.cli.utils module xffl.custom module xffl.custom.config module xffl.custom.formatter module xffl.custom.types module xffl.distributed module xffl.distributed.aggregation module xffl.distributed.distributed module xffl.distributed.distributed_state module xffl.distributed.networking module

xffl.learning module xffl.learning.data module xffl.learning.modelling module xffl.learning.processing module xffl.learning.utils module xffl.utils module xffl.utils.constants module xffl.utils.logging module xffl.utils.utils module xffl.workflow module xffl.workflow.config module xffl.workflow.streamflow module xffl.workflow.templates module xffl.workflow.templates.aggregation_application module xffl.workflow.templates.cwl module xffl.workflow.templates.streamflow module XFFLConfig (class in xffl.custom.config)

Y

YamlConfig (class in xffl.workflow.config)

YELLOW (xffl.custom.formatter.CustomFormatter attribute)