AIO-Data

数据集文件元信息以及数据文件，请浏览“数据集文件”页面获取。 当前数据集卡片使用的是默认模版，数据集的贡献者未提供更加详细的数据集介绍，但是您可以通过如下GIT Clone命令，或者ModelScope SDK来下载数据集 #### 下载方法 :modelscope-code[]{type="sdk"} :modelscope-code[]{type="git"} #### 解压缩 以moire为例： `cd moire` `cat moire.tar.gz.* | tar -xzvf -` 对于haze，直接在根目录解压缩即可 对于val，直接通过`tar -xzvf val.tar.gz`解压缩 #### 将数据整理为以下形式： ``` AIO/ ├── train/ │ ├── moire/ │ │ ├── input/ │ │ └── gt/ │ ├── haze/ │ │ ├── input/ │ │ └── gt/ │ ├── rain/ │ │ ├── input/ │ │ └── gt/ │ ├── snow/ │ | ├── input/ │ | └── gt/ | ├── ... | | └── val/ ├── input/ └── gt/ ``` #### 通过下面的脚本生成csv ``` import os import pandas as pd from tqdm import tqdm def generate_dataset_csv(root_dir, output_csv): label_mapping = { 'noise': 0, 'blur': 1, 'rain': 2, 'over': 3, 'haze': 4, 'raindrop': 6, 'snow': 7, 'under': 8, 'moire': 9, 'low': 10 } data_list = [] if not os.path.exists(root_dir): print(f"Error: Root directory {root_dir} does not exist.") return subfolders = [f for f in os.listdir(root_dir) if os.path.isdir(os.path.join(root_dir, f))] print(f"Scanning directory: {os.path.abspath(root_dir)}") for folder_name in tqdm(subfolders): if folder_name not in label_mapping: print(f"Skipping unknown folder: {folder_name}") continue label = label_mapping[folder_name] folder_path = os.path.join(root_dir, folder_name) input_dir = os.path.join(folder_path, 'input') gt_dir = os.path.join(folder_path, 'gt') if not (os.path.exists(input_dir) and os.path.exists(gt_dir)): print(f"Warning: Missing input or gt in {folder_name}. Skipping.") continue for img_name in os.listdir(input_dir): if not img_name.lower().endswith(('.png', '.jpg', '.jpeg', '.bmp', '.tiff')): continue input_path = os.path.join(input_dir, img_name) gt_path = os.path.join(gt_dir, img_name) if os.path.exists(gt_path): abs_input = os.path.abspath(input_path) abs_gt = os.path.abspath(gt_path) data_list.append([abs_input, abs_gt, label]) else: print(f"Warning: No matching GT for {img_name} in {folder_name}.") df = pd.DataFrame(data_list, columns=['input_path', 'gt_path', 'label']) # Shuffle the dataset to mix labels df = df.sample(frac=1).reset_index(drop=True) df.to_csv(output_csv, index=False) print(f"\nCompleted! Total pairs: {len(data_list)}") print(f"CSV saved to: {os.path.abspath(output_csv)}") if __name__ == "__main__": # Update these paths to your actual environment ROOT_DIR = "./AIO/train" OUTPUT_FILE = "train.csv" generate_dataset_csv(ROOT_DIR, OUTPUT_FILE) ```