######### NON WEB DATA #########
# ~> GUTENBERG BOOKS (5.256 GT)
gutenberg-books,preprocessed/olmo-mix/v1_6-decontaminated/books/{TOKENIZER}/part-0-00000.npy
gutenberg-books,preprocessed/olmo-mix/v1_6-decontaminated/books/{TOKENIZER}/part-1-00000.npy
gutenberg-books,preprocessed/olmo-mix/v1_6-decontaminated/books/{TOKENIZER}/part-2-00000.npy
# ~> PES2O STEM PAPERS (57.21 GT)
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-00-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-01-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-02-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-03-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-04-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-05-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-06-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-07-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-08-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-09-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-10-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-11-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-12-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-13-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-14-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-15-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-16-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-17-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-18-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-19-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-20-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-21-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-22-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-23-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-24-00000.npy
pes2o-stem,preprocessed/olmo-mix/v1_6-decontaminated/pes2o/{TOKENIZER}/part-25-00000.npy
# ~> WIKIPEDIA & WIKIBOOKS (3.689 GT) repeated twice to up-sample
wiki,preprocessed/olmo-mix/v1_6-decontaminated/wiki/{TOKENIZER}/part-0-00000.npy
wiki,preprocessed/olmo-mix/v1_6-decontaminated/wiki/{TOKENIZER}/part-1-00000.npy
wiki,preprocessed/olmo-mix/v1_6-decontaminated/wiki/{TOKENIZER}/part-0-00000.npy
wiki,preprocessed/olmo-mix/v1_6-decontaminated/wiki/{TOKENIZER}/part-1-00000.npy
# MEGAWIKA v1 (4.6 GT)
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-00-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-01-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-01-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-02-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-02-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-03-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-03-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-04-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-04-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-05-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-05-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-06-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-06-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-07-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-07-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-08-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-08-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-09-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-09-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-10-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-10-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-11-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-11-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-12-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-12-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-13-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-13-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-14-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-14-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-15-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-16-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-16-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-17-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-17-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-18-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-18-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-19-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-19-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-20-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-20-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-21-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-21-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-22-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-22-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-23-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-23-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-23-00002.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-24-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-24-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-25-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-25-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-26-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-26-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-27-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-27-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-28-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-29-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-30-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-30-00001.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-31-00000.npy
megawiki,preprocessed/megawika/v1/{TOKENIZER}/part-31-00001.npy
# ~> REDPAJAMA STACK-EXCHANGE (19.63 GT)
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-00-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-01-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-02-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-03-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-04-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-05-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-06-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-07-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-08-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-09-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-10-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-11-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-12-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-13-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-14-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-15-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-16-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-17-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-18-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-19-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-20-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-21-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-22-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-23-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-24-00000.npy
redpajama,preprocessed/redpajama_v1_decon_fix/stackexchange/{TOKENIZER}/part-25-00000.npy
# ~> REDPAJAMA ARXIV (27.97 GT)
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-00-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-01-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-02-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-03-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-04-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-05-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-06-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-07-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-08-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-09-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-10-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-11-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-12-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-13-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-14-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-15-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-16-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-17-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-18-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-19-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-20-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-21-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-22-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-23-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-24-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-25-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-26-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-27-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-28-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-29-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-30-00000.npy
redpajama-arxiv,preprocessed/redpajama_v1_decon_fix/arxiv/{TOKENIZER}/part-31-00000.npy
# ~> PROOFPILE2 ALGEBRAIC STACK (12.623 GT)
proofpile-2-stack,preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/{TOKENIZER}/part-00-00000.npy
proofpile-2-stack,preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/{TOKENIZER}/part-01-00000.npy
proofpile-2-stack,preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/{TOKENIZER}/part-02-00000.npy
proofpile-2-stack,preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/{TOKENIZER}/part-03-00000.npy
proofpile-2-stack,preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/{TOKENIZER}/part-04-00000.npy
proofpile-2-stack,preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/{TOKENIZER}/part-05-00000.npy
proofpile-2-stack,preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/{TOKENIZER}/part-06-00000.npy
proofpile-2-stack,preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/{TOKENIZER}/part-07-00000.npy
proofpile-2-stack,preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/{TOKENIZER}/part-08-00000.npy
proofpile-2-stack,preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/{TOKENIZER}/part-09-00000.npy
proofpile-2-stack,preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/{TOKENIZER}/part-10-00000.npy
proofpile-2-stack,preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/{TOKENIZER}/part-11-00000.npy
proofpile-2-stack,preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/{TOKENIZER}/part-12-00000.npy
proofpile-2-stack,preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/{TOKENIZER}/part-13-00000.npy
proofpile-2-stack,preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/{TOKENIZER}/part-14-00000.npy
proofpile-2-stack,preprocessed/proof-pile-2/v0_decontaminated/algebraic-stack/train/{TOKENIZER}/part-15-00000.npy
# ~> PROOFPILE2 OPENWEBMATH (12.734 GT)
proofpile-2-open-web-math,preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/{TOKENIZER}/part-00-00000.npy
proofpile-2-open-web-math,preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/{TOKENIZER}/part-01-00000.npy
proofpile-2-open-web-math,preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/{TOKENIZER}/part-02-00000.npy
proofpile-2-open-web-math,preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/{TOKENIZER}/part-03-00000.npy
proofpile-2-open-web-math,preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/{TOKENIZER}/part-04-00000.npy
proofpile-2-open-web-math,preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/{TOKENIZER}/part-05-00000.npy
proofpile-2-open-web-math,preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/{TOKENIZER}/part-06-00000.npy
proofpile-2-open-web-math,preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/{TOKENIZER}/part-07-00000.npy
proofpile-2-open-web-math,preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/{TOKENIZER}/part-08-00000.npy
proofpile-2-open-web-math,preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/{TOKENIZER}/part-09-00000.npy
proofpile-2-open-web-math,preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/{TOKENIZER}/part-10-00000.npy
proofpile-2-open-web-math,preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/{TOKENIZER}/part-11-00000.npy
proofpile-2-open-web-math,preprocessed/proof-pile-2/v0_decontaminated/open-web-math/train/{TOKENIZER}/part-12-00000.npy
# ~> TULU FLAN V1 (16.5 G v2-decontaminated-60M-shots_all-upweight_1-dialog_true-sep_newline)
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-00-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-01-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-02-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-03-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-04-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-05-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-06-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-07-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-08-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-09-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-10-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-11-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-12-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-13-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-14-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-15-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-16-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-17-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-18-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-19-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-20-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-21-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-22-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-23-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-24-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-25-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-26-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-27-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-28-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-29-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-30-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-31-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-32-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-33-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-34-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-35-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-36-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-37-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-38-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-39-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-40-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-41-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-42-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-43-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-44-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-45-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-46-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-47-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-48-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-49-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-50-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-51-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-52-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-53-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-54-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-55-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-56-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-57-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-58-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-59-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-60-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-61-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-62-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-63-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-64-00000.npy
tulu-flan-v1,preprocessed/tulu_flan/v2-decontaminated-60M-shots_all-upweight_1-dialog_false-sep_newline/train/{TOKENIZER}/part-65-00000.npy
# ~> CC NEWS (14.3 GT)
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-0-00000.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-0-00001.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-0-00002.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-0-00003.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-1-00000.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-1-00001.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-1-00002.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-1-00003.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-2-00000.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-2-00001.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-2-00002.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-2-00003.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-3-00000.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-3-00001.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-3-00002.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-3-00003.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-4-00000.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-4-00001.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-4-00002.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-4-00003.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-5-00000.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-5-00001.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-6-00000.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-6-00001.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-6-00002.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-6-00003.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-7-00000.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-7-00001.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-7-00002.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-7-00003.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-8-00000.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-8-00001.npy
cc-news,preprocessed/cc-news/v3/{TOKENIZER}/part-8-00002.npy
####################################
######### CODE #########
# ~> STARCODER (263.775 GT)
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-00-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-00-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-01-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-02-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-03-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-03-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-04-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-04-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-05-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-05-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-06-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-06-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-07-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-07-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-08-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-08-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-09-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-09-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-10-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-10-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-11-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-11-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-12-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-12-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-13-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-13-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-14-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-14-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-15-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-15-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-16-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-16-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-17-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-17-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-18-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-18-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-19-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-19-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-20-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-20-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-21-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-21-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-22-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-22-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-23-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-23-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-24-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-24-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-25-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-25-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-26-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-26-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-27-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-27-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-28-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-29-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-30-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-30-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-31-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-31-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-32-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-32-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-33-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-33-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-34-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-34-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-35-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-35-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-36-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-36-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-37-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-37-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-38-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-38-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-39-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-39-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-40-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-40-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-41-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-41-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-42-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-42-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-43-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-43-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-44-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-44-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-45-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-46-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-46-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-47-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-47-00001.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-48-00000.npy
starcoder,preprocessed/starcoder/v0_decontaminated_doc_only/{TOKENIZER}/part-48-00001.npy
####################################
######### WEB HIGH QUALITY #########
# ~> C4 (138.4 GT)
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-000-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-001-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-002-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-003-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-004-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-005-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-006-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-007-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-008-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-009-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-010-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-011-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-012-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-013-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-014-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-015-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-016-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-017-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-018-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-019-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-020-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-021-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-022-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-023-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-024-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-025-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-026-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-027-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-028-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-029-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-030-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-031-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-032-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-033-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-034-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-035-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-036-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-037-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-038-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-039-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-040-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-041-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-042-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-043-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-044-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-045-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-046-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-047-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-048-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-049-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-050-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-051-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-052-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-053-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-054-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-055-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-056-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-057-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-058-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-059-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-060-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-061-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-062-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-063-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-064-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-065-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-066-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-067-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-068-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-069-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-070-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-071-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-072-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-073-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-074-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-075-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-076-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-077-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-078-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-079-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-080-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-081-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-082-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-083-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-084-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-085-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-086-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-087-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-088-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-089-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-090-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-091-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-092-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-093-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-094-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-095-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-096-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-097-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-098-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-099-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-100-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-101-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-102-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-103-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-104-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-105-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-106-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-107-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-108-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-109-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-110-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-111-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-112-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-113-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-114-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-115-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-116-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-117-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-118-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-119-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-120-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-121-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-122-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-123-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-124-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-125-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-126-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-127-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-128-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-129-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-130-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-131-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-132-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-133-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-134-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-135-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-136-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-137-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-138-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-139-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-140-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-141-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-142-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-143-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-144-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-145-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-146-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-147-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-148-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-149-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-150-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-151-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-152-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-153-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-154-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-155-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-156-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-157-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-158-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-159-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-160-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-161-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-162-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-163-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-164-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-165-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-166-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-167-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-168-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-169-00000.npy
c4,preprocessed/c4/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001-fix/{TOKENIZER}/part-170-00000.npy
# ~> REDDIT (79.9 GT)
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-00-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-01-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-02-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-03-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-04-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-05-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-06-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-07-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-08-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-09-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-10-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-11-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-12-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-13-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-14-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-15-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-16-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-17-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-18-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-19-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-20-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-21-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-22-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-23-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-24-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-25-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-26-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-27-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-28-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-29-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-30-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-31-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-32-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-33-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-34-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-35-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-36-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-37-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-38-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-39-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-40-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-41-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-42-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-43-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-44-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-45-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-46-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-47-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-48-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-49-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-50-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-51-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-52-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-53-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-54-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-55-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-56-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-57-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-58-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-59-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-60-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-61-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-62-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-63-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-64-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-65-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-66-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-67-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-68-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-69-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-70-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-71-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-72-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-73-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-74-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-75-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-76-00000.npy
redit,preprocessed/reddit/v5-dedupe-pii-nsfw-toxic-fuzzydd-length/{TOKENIZER}/part-77-00000.npy
# ~> FALCON (547.341 GT)
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-000-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-001-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-002-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-003-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-004-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-005-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-006-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-007-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-008-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-009-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-010-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-011-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-012-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-013-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-014-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-015-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-016-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-017-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-018-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-019-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-020-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-021-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-022-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-023-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-024-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-025-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-026-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-027-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-028-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-029-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-030-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-031-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-032-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-033-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-034-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-035-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-036-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-037-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-038-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-039-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-040-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-041-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-042-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-043-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-044-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-045-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-046-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-047-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-048-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-049-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-050-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-051-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-052-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-053-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-054-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-055-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-056-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-057-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-058-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-059-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-060-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-061-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-062-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-063-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-064-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-065-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-066-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-067-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-068-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-069-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-070-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-071-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-072-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-073-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-074-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-075-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-076-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-077-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-078-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-079-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-080-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-081-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-082-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-083-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-084-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-085-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-086-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-087-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-088-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-089-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-090-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-091-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-092-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-093-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-094-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-095-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-096-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-097-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-098-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-099-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-100-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-101-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-102-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-103-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-104-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-105-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-106-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-107-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-108-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-109-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-110-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-111-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-112-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-113-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-114-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-115-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-116-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-117-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-118-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-119-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-120-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-121-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-122-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-123-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-124-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-125-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-126-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-127-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-128-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-129-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-130-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-131-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-132-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-133-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-134-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-135-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-136-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-137-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-138-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-139-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-140-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-141-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-142-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-143-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-144-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-145-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-146-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-147-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-148-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-149-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-150-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-151-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-152-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-153-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-154-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-155-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-156-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-157-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-158-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-159-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-160-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-161-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-162-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-163-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-164-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-165-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-166-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-167-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-168-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-169-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-170-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-171-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-172-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-173-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-174-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-175-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-176-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-177-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-178-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-179-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-180-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-181-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-182-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-183-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-184-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-185-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-186-00000.npy
falcon,preprocessed/falcon-refinedweb/v2-frac_005_100-qc_cc_multi_bin-paloma-rep-pii/{TOKENIZER}/part-187-00000.npy
####################################
######### WEB REST #########
# ~> DOLMA CC HEAD 50% (178.4 GT)
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-000-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-001-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-002-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-003-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-004-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-005-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-006-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-006-00001.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-007-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-008-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-009-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-010-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-011-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-012-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-013-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-014-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-015-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-016-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-017-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-018-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-019-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-020-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-021-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-022-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-023-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-024-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-025-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-026-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-027-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-028-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-029-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-030-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-031-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-032-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-033-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-034-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-035-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-036-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-037-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-038-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-039-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-040-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-041-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-042-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-043-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-044-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-045-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-046-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-047-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-048-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-049-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-050-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-051-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-052-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-053-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-054-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-055-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-056-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-057-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-058-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-059-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-060-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-061-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-062-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_head/{TOKENIZER}/part-063-00000.npy
# ~> DOLMA CC MIDDLE 33% (242.05 GT)
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-000-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-001-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-002-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-003-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-004-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-005-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-006-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-007-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-008-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-009-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-010-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-011-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-012-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-013-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-014-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-015-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-016-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-017-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-018-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-019-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-020-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-021-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-022-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-023-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-024-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-025-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-026-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-027-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-028-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-029-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-030-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-031-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-032-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-033-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-034-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-035-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-036-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-037-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-038-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-039-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-040-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-041-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-042-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-043-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-044-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-045-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-046-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-047-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-048-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-049-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-050-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-051-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-052-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-053-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-054-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-055-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-056-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-057-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-058-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-059-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-060-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-061-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-062-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-063-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_middle/{TOKENIZER}/part-064-00000.npy
# ~> DOLMA CC TAIL 33% (191.4 GT)
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-000-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-001-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-002-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-003-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-004-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-005-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-006-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-007-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-008-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-009-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-010-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-011-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-012-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-013-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-014-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-015-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-016-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-017-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-018-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-019-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-020-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-021-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-022-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-023-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-024-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-025-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-026-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-027-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-028-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-029-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-030-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-031-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-032-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-033-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-034-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-035-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-036-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-037-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-038-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-039-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-040-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-041-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-042-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-043-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-044-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-045-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-046-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-047-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-048-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-049-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-050-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-051-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-052-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-053-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-054-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-055-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-056-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-057-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-058-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-059-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-060-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-061-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-062-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-063-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-064-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-065-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-066-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-067-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-068-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-069-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-070-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-071-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-072-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-073-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-074-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-075-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-076-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-077-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-078-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-079-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-080-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-081-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-082-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-083-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-084-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-085-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-086-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-087-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-088-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-089-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-090-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-091-00000.npy
dolma-cc,preprocessed/olmo-mix/v1_7-dd_ngram_dp_030-qc_cc_en_bin_001/cc_en_tail/{TOKENIZER}/part-092-00000.npy
