File size: 2,320 Bytes
2fdbaa2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 |
export CORES=`grep -c ^processor /proc/cpuinfo`
export CORES=`echo "scale=0; ${CORES} * 0.8 / 1" | bc`
#export XLA_PYTHON_CLIENT_PREALLOCATE=false
export SOURCE_LANG="en"
export TARGET_LANG="nl"
export HF_PROJECT="ul2-base-neddx2-en-nl"
#
export DATASET="/home/yeb/data/nedd_x_dataset/nedd_x_dataset.py"
#export DATASET_CONFIG="dict"
export DATASET_CONFIG="voc8k_beta_3buf"
export MODEL_NAME_OR_PATH="yhavinga/ul2_base_dutch"
export TOKENIZER_NAME="yhavinga/ul2_base_dutch"
export MODEL_PATH="${HOME}/data/${HF_PROJECT}" # Path to the model
export HF_DATASETS_CACHE=/mnt/ramdisk
# 52k 8k 32ksp
#l 472 500
#b0 328 352
#b1 472 480 370
#b2 1920 1984
mkdir -p ${MODEL_PATH}
python ../run_s2s_flax_pmap_multiseq.py \
--output_dir="${MODEL_PATH}" \
--model_name_or_path ${MODEL_NAME_OR_PATH} \
--tokenizer_name ${TOKENIZER_NAME} \
--use_fast_tokenizer="False" \
--use_auth_token="True" \
--dataset_name_list ${DATASET}\
--dataset_config_name_list "${DATASET_CONFIG}"\
--id_filter_list "<not>-b2-" \
--max_train_samples_list "0" \
--max_eval_samples_list "2000" \
--max_predict_samples_list "128" \
--preprocessing_num_workers="${CORES}" \
--source_lang="${SOURCE_LANG}" \
--target_lang="${TARGET_LANG}" \
--metric_name="sacrebleu" \
--do_train --do_eval --do_predict \
--predict_with_generate \
--learning_rate="0.001" \
--adam_beta1="0.9" \
--adam_beta2="0.9969" \
--adam_epsilon="1e-8" \
--weight_decay="0.001" \
--label_smoothing_factor="0.11" \
--length_penalty="1.3" \
--warmup_steps 500 \
--dropout_rate="0.01" \
--dtype "bfloat16" \
--z_loss "1e-4" \
--dynamic_loss_scaling="False" \
--per_device_train_batch_size 16 \
--per_device_eval_batch_size 16 \
--gradient_accumulation_steps 4 \
--overwrite_output_dir \
--max_source_length_list 370 \
--max_target_length_list 370 \
--num_beams 5 \
--overwrite_output_dir \
--logging_steps 5 \
--save_steps 800 \
--eval_steps 800 \
--num_train_epochs 4 \
--max_eval_samples 512 \
--validation_split_count 2000 \
--wandb_project="${HF_PROJECT}" \
--wandb_job_type="pmap"
# --max_train_samples="1_064_886" \
# --resume_from_checkpoint="${MODEL_PATH}" \
# --max_eval_samples 256 \
# --max_predict_samples 256 \
|