这次的博客对四期模拟赛的预处理操作做一个总结，涉及到机器学习的各种文件的预处理。

传统机器学习数据预处理

import pandas as pd

data = pd.read_csv('songs_origin.csv')
data = data.fillna(data.mean())  # 直接用fillna对缺失值进行填充，缺失值在pandas里表现为NAN值
data = data[(data['acousticness_yr'] <= 1) & (data['acousticness_yr'] >= 0)]  # 反向思考，用索引的方式去除
data = data.drop_duplicates()  # 直接调用去除重复行的attribute

data.to_csv('songs_processed.csv', index=False)

 图像数据预处理（本题使用OpenCV库）

          

这道题只需要完善Resize-Cut-Norm三个操作并编写在img_processor函数中即可，然而我认为这道题更大的价值在于如何处理图像文件数据，代码中给出处理图像的注释。

#task-start
import numpy as np
import json
import cv2
import os

def img_processor(data_path, dst_size = (224,224)):
    Image_std = [0.229, 0.224, 0.225]
    Image_mean = [0.485, 0.456, 0.406]
    _std = np.array(Image_std).reshape((1,1,3))
    _mean = np.array(Image_mean).reshape((1,1,3))
    image_src = cv2.imread(data_path)
    #TODO
    
    # resize操作，可以用cv2的resize，也可以用PIL的resize，这道题由于用CV2读取，PIL的写法更为繁琐
    image = cv2.resize(image_src, (256, 256))
    # 中心裁剪
    startx, starty = (256 - dst_size[0]) // 2, (256 - dst_size[1]) // 2
    image = image[startx:startx + dst_size[0], starty:starty+dst_size[1], :]
    # 中心化
    image = (image - _mean) / _std
    return image_src, image, (startx,starty)


def simple_generator(data_list, json_file, dst_size = (224, 224)):
    with open(json_file, 'r') as f:
        data = json.load(f)
    folder_map = {v[0]: (int(k), v[1]) for k,v in data.items()}

    for img_path in data_list:
        image_src, image, (startx,starty) = img_processor(img_path, dst_size)
        label = folder_map[img_path.split('/')[-2]]
        yield image_src, image, label, (startx,starty)

def main():
    Image_path = 'Imagedata/images'
    Json_path = 'Imagedata/image_class_index.json'   # json文件读取后为字典，key为标签索引[0...n]，Value为标签代号以及class name
    data_list = []
    for dirname, _, filenames in os.walk(Image_path):  # os.walk输出三元组，dirpath， dirnames, files
        # 例如文件为 'Imagedata/images/1/4.jpg' 以及 'Imagedata/images/2/3.jpg', walk输出则为['Imagedata/images/1', ['Imagedata/images/2']， ['1', '2'], ['4.jpg', '3.jpg']
        # 上述注释仅为便于理解
        if os.path.basename(dirname).startswith('n'):  # 判断dir文件夹是否为n开头，标签代号都为n开头
            for filename in filenames:
                data_list.append(os.path.join(dirname, filename)) 

    # 创建生成器
    generator = simple_generator(data_list, Json_path)  # 用yield创建生成器

    # 查看示例，检查图像、标签等属性的正确性
    num_samples = 5
    for _ in range(num_samples):
        image_src, image, label, (startx,starty) = next(generator)
        print("SrcImage shape:", image_src.shape)
        print("Image shape:", image.shape)
        print("Label:", label)
        print("startx and starty:",(startx,starty))

if __name__ == '__main__':
    main()
#task-end

 文本同义词替换

#task-start
import random
import pandas as pd
from torch.utils.data import Dataset


class MakeDataset(Dataset):

    def __init__(self):
        self.data = pd.read_csv('data.csv')[['text_id', 'text']].values
        self.locs = open('loc.txt', 'r').read().split('\n')
        self.pers = open('per.txt', 'r').read().split('\n')

    def __getitem__(self, item):
        text_id, text = self.data[item]
        text, aug_info = self.augment(text)
        return text_id, text, aug_info

    def __len__(self):
        return len(self.data)

    def augment(self, text):
        aug_info = {'locs': [], 'pers': []}

        # TODO
        loc_temp, per_temp = [], []   # 存储text中有多少个需要替换的词
        for loc in self.locs:
            if loc in text:
                loc_temp.append(loc)
        for loc in loc_temp:
            loc_replace = random.choice(self.locs)
            text = text.replace(loc,loc_replace)
            aug_info['locs'].append({'original':loc,'replacement':loc_replace}) 
        for per in self.pers:
            if per in text:
                per_temp.append(per)
        for per in per_temp:
            per_replace = random.choice(self.pers)
            text = text.replace(per,per_replace)
            aug_info['pers'].append({'original':per,'replacement':per_replace})
        return text, aug_info


def main():
    dataset = MakeDataset()
    for data in dataset:
        print(data)

if __name__ == '__main__':
    main()
#task-end

这道题代码写的繁琐了，后续可以优化一下代码