企业官网建设流程全解析

● 🍨 本文为🔗365天深度学习训练营中的学习记录博客

● 🍖 原作者：K同学啊

一、前期准备

1.设置GPU

import tensorflow as tf gpus = tf.config.list_physical_devices("GPU") if gpus: tf.config.experimental.set_memory_growth(gpus[0], True) #设置GPU显存用量按需使用 tf.config.set_visible_devices([gpus[0]],"GPU")

2.导入数据

import numpy as np import matplotlib.pyplot as plt # 支持中文 plt.rcParams['font.sans-serif'] = ['SimHei'] plt.rcParams['axes.unicode_minus'] = False import os,PIL,pathlib #隐藏警告 import warnings warnings.filterwarnings('ignore') data_dir = "D:/新建文件夹/365-7-data" data_dir = pathlib.Path(data_dir) image_count = len(list(data_dir.glob('*/*'))) print("图片总数为：",image_count)

显示有3400张图片

3.加载数据

batch_size = 64 img_height = 224 img_width = 224 train_ds = tf.keras.preprocessing.image_dataset_from_directory( data_dir, validation_split=0.2, subset="training", seed=12, image_size=(img_height, img_width), batch_size=batch_size) val_ds = tf.keras.preprocessing.image_dataset_from_directory( data_dir, validation_split=0.2, subset="validation", seed=12, image_size=(img_height, img_width), batch_size=batch_size)

4.分类名

class_names = train_ds.class_names print(class_names)

显示有cat和dog两种

5.再次检查数据

for image_batch, labels_batch in train_ds: print(image_batch.shape) print(labels_batch.shape) break

显示（64，224，224，3）

（64，）

6.配置数据集

AUTOTUNE = tf.data.AUTOTUNE def preprocess_image(image,label): return (image/255.0,label) # 归一化处理 train_ds = train_ds.map(preprocess_image, num_parallel_calls=AUTOTUNE) val_ds = val_ds.map(preprocess_image, num_parallel_calls=AUTOTUNE) train_ds = train_ds.cache().shuffle(1000).prefetch(buffer_size=AUTOTUNE) val_ds = val_ds.cache().prefetch(buffer_size=AUTOTUNE)

7.数据可视化

plt.figure(figsize=(15, 10)) # 图形的宽为15高为10 for images, labels in train_ds.take(1): for i in range(8): ax = plt.subplot(5, 8, i + 1) plt.imshow(images[i]) plt.title(class_names[labels[i]]) plt.axis("off")

二、建立VGG-16模型

from tensorflow.keras import layers, models, Input from tensorflow.keras.models import Model from tensorflow.keras.layers import Conv2D, MaxPooling2D, Dense, Flatten, Dropout def VGG16(nb_classes, input_shape): input_tensor = Input(shape=input_shape) # 1st block x = Conv2D(64, (3,3), activation='relu', padding='same',name='block1_conv1')(input_tensor) x = Conv2D(64, (3,3), activation='relu', padding='same',name='block1_conv2')(x) x = MaxPooling2D((2,2), strides=(2,2), name = 'block1_pool')(x) # 2nd block x = Conv2D(128, (3,3), activation='relu', padding='same',name='block2_conv1')(x) x = Conv2D(128, (3,3), activation='relu', padding='same',name='block2_conv2')(x) x = MaxPooling2D((2,2), strides=(2,2), name = 'block2_pool')(x) # 3rd block x = Conv2D(256, (3,3), activation='relu', padding='same',name='block3_conv1')(x) x = Conv2D(256, (3,3), activation='relu', padding='same',name='block3_conv2')(x) x = Conv2D(256, (3,3), activation='relu', padding='same',name='block3_conv3')(x) x = MaxPooling2D((2,2), strides=(2,2), name = 'block3_pool')(x) # 4th block x = Conv2D(512, (3,3), activation='relu', padding='same',name='block4_conv1')(x) x = Conv2D(512, (3,3), activation='relu', padding='same',name='block4_conv2')(x) x = Conv2D(512, (3,3), activation='relu', padding='same',name='block4_conv3')(x) x = MaxPooling2D((2,2), strides=(2,2), name = 'block4_pool')(x) # 5th block x = Conv2D(512, (3,3), activation='relu', padding='same',name='block5_conv1')(x) x = Conv2D(512, (3,3), activation='relu', padding='same',name='block5_conv2')(x) x = Conv2D(512, (3,3), activation='relu', padding='same',name='block5_conv3')(x) x = MaxPooling2D((2,2), strides=(2,2), name = 'block5_pool')(x) # full connection x = Flatten()(x) x = Dense(4096, activation='relu', name='fc1')(x) x = Dense(4096, activation='relu', name='fc2')(x) output_tensor = Dense(nb_classes, activation='softmax', name='predictions')(x) model = Model(input_tensor, output_tensor) return model model=VGG16(1000, (img_width, img_height, 3)) model.summary()

三、编译

model.compile(optimizer="adam", loss ='sparse_categorical_crossentropy', metrics =['accuracy'])

四、训练

from tqdm import tqdm import tensorflow.keras.backend as K epochs = 10 lr = 1e-4 # 记录训练数据，方便后面的分析 history_train_loss = [] history_train_accuracy = [] history_val_loss = [] history_val_accuracy = [] for epoch in range(epochs): train_total = len(train_ds) val_total = len(val_ds) """ total：预期的迭代数目 ncols：控制进度条宽度 mininterval：进度更新最小间隔，以秒为单位（默认值：0.1） """ with tqdm(total=train_total, desc=f'Epoch {epoch + 1}/{epochs}',mininterval=1,ncols=100) as pbar: lr = lr*0.92 K.set_value(model.optimizer.lr, lr) train_loss = [] train_accuracy = [] for image,label in train_ds: """ 训练模型，简单理解train_on_batch就是：它是比model.fit()更高级的一个用法 想详细了解 train_on_batch 的同学， 可以看看我的这篇文章：https://www.yuque.com/mingtian-fkmxf/hv4lcq/ztt4gy """ # 这里生成的是每一个batch的acc与loss history = model.train_on_batch(image,label) train_loss.append(history[0]) train_accuracy.append(history[1]) pbar.set_postfix({"train_loss": "%.4f"%history[0], "train_acc":"%.4f"%history[1], "lr": K.get_value(model.optimizer.lr)}) pbar.update(1) history_train_loss.append(np.mean(train_loss)) history_train_accuracy.append(np.mean(train_accuracy)) print('开始验证！') with tqdm(total=val_total, desc=f'Epoch {epoch + 1}/{epochs}',mininterval=0.3,ncols=100) as pbar: val_loss = [] val_accuracy = [] for image,label in val_ds: # 这里生成的是每一个batch的acc与loss history = model.test_on_batch(image,label) val_loss.append(history[0]) val_accuracy.append(history[1]) pbar.set_postfix({"val_loss": "%.4f"%history[0], "val_acc":"%.4f"%history[1]}) pbar.update(1) history_val_loss.append(np.mean(val_loss)) history_val_accuracy.append(np.mean(val_accuracy)) print('结束验证！') print("验证loss为：%.4f"%np.mean(val_loss)) print("验证准确率为：%.4f"%np.mean(val_accuracy))

from datetime import datetime current_time = datetime.now() # 获取当前时间 epochs_range = range(epochs) plt.figure(figsize=(12, 4)) plt.subplot(1, 2, 1) plt.plot(epochs_range, history_train_accuracy, label='Training Accuracy') plt.plot(epochs_range, history_val_accuracy, label='Validation Accuracy') plt.legend(loc='lower right') plt.title('Training and Validation Accuracy') plt.xlabel(current_time) plt.subplot(1, 2, 2) plt.plot(epochs_range, history_train_loss, label='Training Loss') plt.plot(epochs_range, history_val_loss, label='Validation Loss') plt.legend(loc='upper right') plt.title('Training and Validation Loss') plt.show()

五、对T8纠错

以上的代码是改正了T8错误的代码

1.batch_size

T8中batch_size为8，正确代码为64

但我的显卡内存不足，想增大batch_size时会报如下错：

所以我的最终结果可能会有一些不符

batch_size大时可以用更多的样本计算平均梯度，梯度方向更准，震荡会减少

2.acc和loss的计算

这个问题才是主要bug

在T8中是这样计算acc和loss的：

history = model.train_on_batch(image,label) train_loss = history[0] train_accuracy = history[1] pbar.set_postfix({"loss": "%.4f"%train_loss, "accuracy":"%.4f"%train_accuracy, "lr": K.get_value(model.optimizer.lr)}) pbar.update(1) history_train_loss.append(train_loss) history_train_accuracy.append(train_accuracy)

和本周进行对比

train_loss = [] train_accuracy = [] for image,label in train_ds: history = model.train_on_batch(image,label) train_loss.append(history[0]) train_accuracy.append(history[1]) pbar.set_postfix({"train_loss": "%.4f"%history[0], "train_acc":"%.4f"%history[1], "lr": K.get_value(model.optimizer.lr)}) pbar.update(1) history_train_loss.append(np.mean(train_loss) history_train_accuracy.append(np.mean(train_accuracy))

应该对train_loss和train_accuracy分别求平均才能得到本轮的history_train_loss和history_train_accuracy，而T8只能保留一轮中的最后一个train_loss和train_accuracy作为本轮的history_train_accuracy，不具有代表性，验证集也同理。

个人总结：本周按照K同学呀老师提供的代码对T8进行纠错，T8中非常严重的bug是loss和accuracy的计算错误，accuracy是指标，它的值出现问题会影响训练优化的方向。另外，T8和T9使用train_on_batch进行训练，与之前使用的代码简洁的fit相比，train_on_batch更加灵活，需要另外写出外层的训练循环，可以对训练过程进行更加精细的控制。