cleaning on train.py and added script for dummy pretraining without deepspeed

llava/train/train.py

@@ -803,7 +803,7 @@ class LazySupervisedDataset(Dataset):
             lmdb_key = self.list_data_dict[i]['image']
             #image_folder = self.data_args.image_folder
             processor = self.data_args.image_processor
-            #image = Image.open(os.path.join(image_folder, image_file)).convert('RGB') # TODO convert('RGB')!?!?
+            #image = Image.open(os.path.join(image_folder, image_file)).convert('RGB')
             # label is BEN multi-label and not needed for this task
             image, _ = self.BENv2Loader[lmdb_key]
             if self.image_size[0] == 14:
@@ -827,7 +827,7 @@ class LazySupervisedDataset(Dataset):
                 # not needed, image is already square        
                 #image = expand2square(image, tuple(int(x*255) for x in processor.image_mean))
                 
-                image = processor.preprocess(image, return_tensors='pt')['pixel_values'][0] # FIXME Adapt config of image_processor
+                image = processor.preprocess(image, return_tensors='pt')['pixel_values'][0]
             else:
                 image = processor.preprocess(image, return_tensors='pt', input_data_format='channels_first')['pixel_values'][0] # add channels_first because it can only infer channels for 1 / 3 bands
             
@@ -842,7 +842,7 @@ class LazySupervisedDataset(Dataset):
             sources = copy.deepcopy([e["conversations"] for e in sources])
         
         # sources contains only conversation, not the image
-        data_dict = preprocess( # TODO Look into function
+        data_dict = preprocess(
             sources,
             self.tokenizer,
             has_image=('image' in self.list_data_dict[i]))
@@ -910,9 +910,6 @@ def make_supervised_data_module(tokenizer: transformers.PreTrainedTokenizer,
                                 data_path=data_args.data_path,
                                 data_args=data_args)
 
-    test = train_dataset[0]
-
-    # FIXME error in preprocessor.preprocess because of values being out of range to convert in uint8
     data_collator = DataCollatorForSupervisedDataset(tokenizer=tokenizer)
     return dict(train_dataset=train_dataset,
                 eval_dataset=None,

scripts/v1_5/pretrain_dummy_ms_mars_no_deepspeed.sh

+python ./llava/train/train_mem.py \
+    --model_name_or_path ../../../mnt/storagecube/johann/llava-v1.5-vit-s \
+    --version plain \
+    --data_path ./BEN_10.json \
+    --image_folder ./playground \
+    --vision_tower ./clip-S-224-16-zavras/ \
+    --mm_projector_type linear \
+    --tune_mm_mlp_adapter True \
+    --mm_vision_select_layer -2 \
+    --mm_use_im_start_end False \
+    --mm_use_im_patch_token False \
+    --bf16 True \
+    --output_dir ../../../mnt/storagecube/johann/checkpoints/llava-v1.5-7b-vit-s-pretrain-dummy \
+    --num_train_epochs 5 \
+    --per_device_train_batch_size 2 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 1 \
+    --evaluation_strategy "no" \
+    --save_strategy "no" \
+    --save_steps 24000 \
+    --save_total_limit 1 \
+    --learning_rate 1e-3 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 True \
+    --model_max_length 2048 \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 4 \
+    --lazy_preprocess True \
+    --report_to wandb \
+    --ben_lmdb_path ../../../data/kaiclasen/BENv2.lmdb \
+    --ben_meta_parquet ../../../data/kaiclasen/metadata.parquet \
+    --ben_snow_cloud_parquet ../../../data/kaiclasen/metadata_for_patches_with_snow_cloud_or_shadow.parquet \
+    --ben_image_size 224 \
+    --ben_channels 14