upload

vlm/merge_and_upload.py

+from datasets import load_dataset, DatasetDict, concatenate_datasets
+
+### 1번 데이터
+ds = load_dataset('kihoonlee/Korean-IC2024-processed', trust_remote_code=True)
+train_ds = ds["train"]
+val_ds = ds["dev"]
+test_ds = ds["test"]
+
+print(train_ds)
+print(val_ds)
+print(test_ds)
+
+def transform_dataset1(dataset):
+    def transform_example(example):
+        return {
+            'source': 'Korean-IC2024',
+            'questions': "이미지를 설명해주세요.",
+            'answers': example['output'],
+            'images': example['image']
+        }
+    return dataset.map(transform_example, batch_size=10000)
+
+a = transform_dataset1(train_ds)
+b = transform_dataset1(val_ds)
+c = transform_dataset1(test_ds)
+
+print(a)
+print(b)
+print(c)
+
+### 2번 데이터
+ds = load_dataset('kihoonlee/KoLLaVA-Instruct-313k', trust_remote_code=True)
+d = ds["train"].map(lambda example: {**example, 'questions': example['questions'].replace('<image>', '').strip()}, batch_size=10000)
+
+print(d)
+
+### 3번 데이터
+ds = load_dataset('kms7530/ko-coco-bal', trust_remote_code=True)
+train_ds = ds["train"]
+val_ds = ds["validation"]
+print(train_ds)
+print(val_ds)
+
+def transform_dataset2(dataset):
+    def transform_example(example):
+        return {
+            'source': 'kms7530/ko-coco-bal',
+            'questions': "이미지를 설명해주세요.",
+            'answers': example['caption_ko'],
+            'images': example['image']
+        }
+    return dataset.map(transform_example, batch_size=10000)
+
+e = transform_dataset2(train_ds)
+f = transform_dataset2(val_ds)
+
+print(e)
+print(f)
+
+### Merge & Upload
+all_data = concatenate_datasets([a,b,c,d,e,f])
+all_dataset = DatasetDict({"train": all_data})
+
+all_dataset.push_to_hub("kihoonlee/vlm_kotuning_dataset", private=True)
+
+print("Dataset uploaded to Hugging Face Hub successfully.")
\ No newline at end of file