深度学习入门数据集--2.fasion-mnist数据集

Fasion-MNIST是一位老师推荐给我的，要求我在做完MNIST-handwriting数据之后，再玩一下fmnist。这个数据集也是28*28的灰度图像，你可以从下面的图片看清图片内容。便签也是10类，恤、裤子、套衫、裙子、外套、凉鞋、汗衫、运动鞋、包、踝靴。这个数据集是由一家德国的时尚科技公司Zalando提供的，我觉一家公司把定位成时尚科技公司，而不是电商平台，是把科技创新能力作为主要生产力。未来，会更多如此定位的公司，医疗科技公司、美食科技公司、出行科技公司、建筑科技公司~。本文主要用Keras编写模型，训练数据，并以清晰的可视化方式展示。

查看数据

数据可以从git仓库上下载，https://github.com/zalandoresearch/fashion-mnist
fasion-mnist 作为tensorflow分类教程数据，通过访问链接可以获得更多内容https://www.tensorflow.org/tutorials/keras/basic_classification

观察数据

代码：

import numpy
import matplotlib.pyplot as plt
import gzip

IMAGE_SIZE = 28
NUM_CHANNELS = 1
PIXEL_DEPTH = 255
NUM_LABELS = 10
def extract_data(filename, num_images):
    print('Extracting', filename)
    with gzip.open(filename) as bytestream:
        bytestream.read(16)
        buf = bytestream.read(IMAGE_SIZE * IMAGE_SIZE * num_images * NUM_CHANNELS)
        data = numpy.frombuffer(buf, dtype=numpy.uint8).astype(numpy.float32)
        data = numpy.reshape(data, [num_images, -1])
        return data
def extract_labels(filename, num_images):
    """Extract the labels into a vector of int64 label IDs."""
    print('Extracting', filename)
    with gzip.open(filename) as bytestream:
        bytestream.read(8)
        buf = bytestream.read(1 * num_images)
        labels = numpy.frombuffer(buf, dtype=numpy.uint8).astype(numpy.int64)
        return labels
        
path= '/Users/wangsen/ai/13/homework/fmnist'
train_data = extract_data(path+'/train-images-idx3-ubyte.gz', 60000)
train_labels = extract_labels(path+'/train-labels-idx1-ubyte.gz', 60000)
test_data = extract_data(path+'/t10k-images-idx3-ubyte.gz', 10000)
test_labels = extract_labels(path+'/t10k-labels-idx1-ubyte.gz', 10000)
labels = ['T恤','裤子','套衫','裙子','外套','凉鞋','汗衫','运动鞋','包','踝靴']

pic = train_data[10]
pic = pic.reshape(28,28)
k = 0
fig, axes1 = plt.subplots(10,10,figsize=(6,7.5))
for i in range(10):
    for j in range(10):
        axes1[i][j].set_axis_off()
        pic = train_data[k]
        pic = pic.reshape(28,28)
        axes1[i][j].imshow(pic,cmap='gray')
        axes1[i][j].set_title(labels[train_labels[k]],fontsize=9)
        k = k+1
plt.show()

Keras 构造Medal

• 运行并训练模型，代码源于Google Colab，是一个资源非常好的站点。
  https://colab.research.google.com/github/margaretmz/deep-learning/blob/master/fashion_mnist_keras.ipynb
  运行以上程序，10分钟会验证集准确率能达到90%。

import numpy as np
import matplotlib.pyplot as plt
import gzip
import os

import numpy as np

paths = [
    '/Users/wangsen/ai/13/homework/fmnist/train-labels-idx1-ubyte.gz', '/Users/wangsen/ai/13/homework/fmnist/train-images-idx3-ubyte.gz',
    '/Users/wangsen/ai/13/homework/fmnist/t10k-labels-idx1-ubyte.gz', '/Users/wangsen/ai/13/homework/fmnist/t10k-images-idx3-ubyte.gz'
]

with gzip.open(paths[0], 'rb') as lbpath:
    y_train = np.frombuffer(lbpath.read(), np.uint8, offset=8)

with gzip.open(paths[1], 'rb') as imgpath:
    x_train = np.frombuffer(imgpath.read(), np.uint8, offset=16).reshape(len(y_train), 28, 28)

with gzip.open(paths[2], 'rb') as lbpath:
    y_test = np.frombuffer(lbpath.read(), np.uint8, offset=8)

with gzip.open(paths[3], 'rb') as imgpath:
    x_test = np.frombuffer(imgpath.read(), np.uint8, offset=16).reshape(len(y_test), 28, 28)
print("x_train shape:", x_train.shape, "y_train shape:", y_train.shape)

# Print the number of training and test datasets
print(x_train.shape[0], 'train set')
print(x_test.shape[0], 'test set')

# Define the text labels
# fashion_mnist_labels = ["T-shirt/top",  # index 0
#                         "Trouser",      # index 1
#                         "Pullover",     # index 2 
#                         "Dress",        # index 3 
#                         "Coat",         # index 4
#                         "Sandal",       # index 5
#                         "Shirt",        # index 6 
#                         "Sneaker",      # index 7 
#                         "Bag",          # index 8 
#                         "Ankle boot"]   # index 9
fashion_mnist_labels=['T恤','裤子','套衫','裙子','外套','凉鞋','汗衫','运动鞋','包','踝靴']
# Image index, you can pick any number between 0 and 59,999
img_index = 5
# y_train contains the lables, ranging from 0 to 9
label_index = y_train[img_index]
# Print the label, for example 2 Pullover
print ("y = " + str(label_index) + " " +(fashion_mnist_labels[label_index]))
# # Show one of the images from the training dataset


x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255

# Further break training data into train / validation sets (# put 5000 into validation set and keep remaining 55,000 for train)
(x_train, x_valid) = x_train[5000:], x_train[:5000] 
(y_train, y_valid) = y_train[5000:], y_train[:5000]

# Reshape input data from (28, 28) to (28, 28, 1)
w, h = 28, 28
x_train = x_train.reshape(x_train.shape[0], w, h, 1)
x_valid = x_valid.reshape(x_valid.shape[0], w, h, 1)
x_test = x_test.reshape(x_test.shape[0], w, h, 1)

# One-hot encode the labels
y_train = tf.keras.utils.to_categorical(y_train, 10)
y_valid = tf.keras.utils.to_categorical(y_valid, 10)
y_test = tf.keras.utils.to_categorical(y_test, 10)

# Print training set shape
print("x_train shape:", x_train.shape, "y_train shape:", y_train.shape)

# Print the number of training, validation, and test datasets
print(x_train.shape[0], 'train set')
print(x_valid.shape[0], 'validation set')
print(x_test.shape[0], 'test set')

model = tf.keras.Sequential()

# Must define the input shape in the first layer of the neural network
model.add(tf.keras.layers.Conv2D(filters=64, kernel_size=2, padding='same', activation='relu', input_shape=(28,28,1))) 
model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
model.add(tf.keras.layers.Dropout(0.3))

model.add(tf.keras.layers.Conv2D(filters=32, kernel_size=2, padding='same', activation='relu'))
model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
model.add(tf.keras.layers.Dropout(0.3))

model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(256, activation='relu'))
model.add(tf.keras.layers.Dropout(0.5))
model.add(tf.keras.layers.Dense(10, activation='softmax'))

# Take a look at the model summary
model.summary()

model.compile(loss='categorical_crossentropy',
             optimizer='adam',
             metrics=['accuracy'])

from keras.callbacks import ModelCheckpoint

checkpointer = ModelCheckpoint(filepath='model.weights.best.hdf5', verbose = 1, save_best_only=True)
model.fit(x_train,
         y_train,
         batch_size=64,
         epochs=10,
         validation_data=(x_valid, y_valid),
         callbacks=[checkpointer])
# Evaluate the model on test set
score = model.evaluate(x_test, y_test, verbose=0)

• 测试
  从代码可以看出，用Keras编写代码，建模、保存参数等代码量都较小。最后的结果显示，15个例子中有2个错误例子，一个“T恤”识别成“包”，一个把“汗衫”识别成“外套”。

  
  预测结果

import tensorflow as tf

import numpy as np
import matplotlib.pyplot as plt
import gzip
import os

import numpy as np

paths = [
    '/Users/wangsen/ai/13/homework/fmnist/train-labels-idx1-ubyte.gz', '/Users/wangsen/ai/13/homework/fmnist/train-images-idx3-ubyte.gz',
    '/Users/wangsen/ai/13/homework/fmnist/t10k-labels-idx1-ubyte.gz', '/Users/wangsen/ai/13/homework/fmnist/t10k-images-idx3-ubyte.gz'
]

with gzip.open(paths[0], 'rb') as lbpath:
    y_train = np.frombuffer(lbpath.read(), np.uint8, offset=8)

with gzip.open(paths[1], 'rb') as imgpath:
    x_train = np.frombuffer(imgpath.read(), np.uint8, offset=16).reshape(len(y_train), 28, 28)

with gzip.open(paths[2], 'rb') as lbpath:
    y_test = np.frombuffer(lbpath.read(), np.uint8, offset=8)

with gzip.open(paths[3], 'rb') as imgpath:
    x_test = np.frombuffer(imgpath.read(), np.uint8, offset=16).reshape(len(y_test), 28, 28)
print("x_train shape:", x_train.shape, "y_train shape:", y_train.shape)

# Print the number of training and test datasets
print(x_train.shape[0], 'train set')
print(x_test.shape[0], 'test set')
fashion_mnist_labels=['T恤','裤子','套衫','裙子','外套','凉鞋','汗衫','运动鞋','包','踝靴']
# Image index, you can pick any number between 0 and 59,999
img_index = 5
# y_train contains the lables, ranging from 0 to 9
label_index = y_train[img_index]
# Print the label, for example 2 Pullover
print ("y = " + str(label_index) + " " +(fashion_mnist_labels[label_index]))
# # Show one of the images from the training dataset

x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255

# Further break training data into train / validation sets (# put 5000 into validation set and keep remaining 55,000 for train)
(x_train, x_valid) = x_train[5000:], x_train[:5000] 
(y_train, y_valid) = y_train[5000:], y_train[:5000]

# Reshape input data from (28, 28) to (28, 28, 1)
w, h = 28, 28
x_train = x_train.reshape(x_train.shape[0], w, h, 1)
x_valid = x_valid.reshape(x_valid.shape[0], w, h, 1)
x_test = x_test.reshape(x_test.shape[0], w, h, 1)

# One-hot encode the labels
y_train = tf.keras.utils.to_categorical(y_train, 10)
y_valid = tf.keras.utils.to_categorical(y_valid, 10)
y_test = tf.keras.utils.to_categorical(y_test, 10)

# Print training set shape
print("x_train shape:", x_train.shape, "y_train shape:", y_train.shape)

# Print the number of training, validation, and test datasets
print(x_train.shape[0], 'train set')
print(x_valid.shape[0], 'validation set')
print(x_test.shape[0], 'test set')

model = tf.keras.Sequential()

# Must define the input shape in the first layer of the neural network
model.add(tf.keras.layers.Conv2D(filters=64, kernel_size=2, padding='same', activation='relu', input_shape=(28,28,1))) 
model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
model.add(tf.keras.layers.Dropout(0.3))

model.add(tf.keras.layers.Conv2D(filters=32, kernel_size=2, padding='same', activation='relu'))
model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
model.add(tf.keras.layers.Dropout(0.3))

model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(256, activation='relu'))
model.add(tf.keras.layers.Dropout(0.5))
model.add(tf.keras.layers.Dense(10, activation='softmax'))

from keras.callbacks import ModelCheckpoint

checkpointer = ModelCheckpoint(filepath='model.weights.best.hdf5', verbose = 1, save_best_only=True)

model.load_weights(filepath='model.weights.best.hdf5')
y_hat = model.predict(x_test)


figure = plt.figure(figsize=(20, 8))
for i, index in enumerate(np.random.choice(x_test.shape[0], size=15, replace=False)):
    ax = figure.add_subplot(3, 5, i + 1, xticks=[], yticks=[])
    # Display each image
    ax.imshow(np.squeeze(x_test[index]))
    predict_index = np.argmax(y_hat[index])
    true_index = np.argmax(y_test[index])
    # Set the title for each image
    ax.set_title("{} ({})".format(fashion_mnist_labels[predict_index], 
                                  fashion_mnist_labels[true_index]),
                                  color=("green" if predict_index == true_index else "red"))
plt.show()

官网图像嵌入原图，大家一起欣赏一下。

图像嵌入