LICENSE
README.md
pyproject.toml
data_forager/__init__.py
data_forager/sample_index.py
data_forager/unzip_files.py
data_forager/utils.py
data_forager.egg-info/PKG-INFO
data_forager.egg-info/SOURCES.txt
data_forager.egg-info/dependency_links.txt
data_forager.egg-info/requires.txt
data_forager.egg-info/top_level.txt
data_forager/datasets/__init__.py
data_forager/datasets/common.py
data_forager/datasets/jsonl.py
data_forager/datasets/tokens.py
data_forager/datasets/tokens_with_aux.py
data_forager/index_stores/__init__.py
data_forager/index_stores/common.py
data_forager/index_stores/fs_based.py
data_forager/indexers/__init__.py
data_forager/indexers/jsonl_indexer.py
data_forager/indexers/text_lines.py
data_forager/indexers/tokenization_indexer.py
data_forager/sample_generators/__init__.py
data_forager/sample_generators/common.py
data_forager/sample_generators/schema.py
data_forager/sample_generators/tokenization.py
data_forager/sample_generators/tokenization_with_aux.py
data_forager/sample_generators/aux/__init__.py
data_forager/sample_generators/aux/common.py
data_forager/sample_generators/aux/loss_mask.py
tests/test_jsonl_indexing.py
tests/test_tokenization_with_aux.py
tests/test_tokenizing_indexing_jsonl.py