import pandas as pd
import cv2
import os
train = pd.read_csv('train.csv')
os.mkdir('train_folders')
os.mkdir('validation_folders')
for i in range(10):
images = train['id'][train['label'] == i].tolist()
num_images = len(images)
os.mkdir('train_folders/' + str(i))
for image in images[:int(0.85*num_images)]:
img = cv2.imread('train/' + str(image)+ '.png')
cv2.imwrite('train_folders/' + str(i) + '/' + str(image) + '.png', img)
os.mkdir('validation_folders/' + str(i))
for image in images[int(0.85*num_images):]:
img = cv2.imread('train/' + str(image)+ '.png')
cv2.imwrite('validation_folders/' + str(i) + '/' + str(image) + '.png', img)
cf sapml fs put train_folders fashion_mnist/
cf sapml fs put validation_folders fashion_mnist/
## necessary imports
import pandas as pd
import numpy as np
import keras
from keras.preprocessing.image import ImageDataGenerator
from keras.applications.inception_resnet_v2 import preprocess_input
from keras.models import Model, load_model
from keras.callbacks import ModelCheckpoint, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau
from keras import optimizers, losses, activations, models
from keras.layers import Convolution2D, Dense, Input, Flatten, Dropout, MaxPooling2D, BatchNormalization, GlobalMaxPool2D
## some constants
SIZE = 28
NUM_CLASSES = 10
BATCH_SIZE = 16
SEED = 1
train_data_dir = '../../fashion_mnist/train_folders/'
validation_data_dir = '../../fashion_mnist/validation_folders/'
nb_train_samples = 51000
nb_validation_samples = 9000
image_gen = ImageDataGenerator(rotation_range=15,
width_shift_range=0.2,
height_shift_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
rescale=1./255,
preprocessing_function=preprocess_input)
val_image_gen = ImageDataGenerator(rescale=1./255, preprocessing_function=preprocess_input)
train_gen = image_gen.flow_from_directory(train_data_dir, batch_size=BATCH_SIZE, seed=SEED, target_size=(SIZE, SIZE), shuffle=True, class_mode = "categorical")
valid_gen = val_image_gen.flow_from_directory(validation_data_dir, batch_size=BATCH_SIZE, seed=SEED, target_size=(SIZE, SIZE), shuffle=True, class_mode = "categorical")
def get_model():
keras.backend.clear_session()
model = keras.applications.inception_resnet_v2.InceptionResNetV2(include_top=False,
weights='imagenet', input_tensor=None, input_shape=(SIZE, SIZE, 3), pooling='avg')
y = Dense(32, activation='relu')(model.layers[-1].output)
y = Dropout(0.3)(y)
y = Dense(32, activation='relu')(y)
y = Dropout(0.2)(y)
y = Dense(NUM_CLASSES, activation='softmax')(y)
model = Model(inputs=model.input, outputs=y)
for l in model.layers[:-1]:
l.trainable = False
sgd = keras.optimizers.SGD(lr=0.001, momentum=0.9)
model.compile(optimizer=sgd, loss=losses.categorical_crossentropy, metrics=['acc'])
model.summary()
return model
filepath = '../../fashion_mnist/model-inception-resnet-v2-retrain-{epoch:02d}-{val_loss:.2f}-{val_acc:.4f}.hdf5'
checkpointer = ModelCheckpoint(filepath, verbose=1, save_best_only=True)
early = EarlyStopping(monitor='val_acc', min_delta=0, patience=3, verbose=1, mode='auto')
reduce_lr = ReduceLROnPlateau(monitor='val_acc', factor=0.1, patience=2, min_lr=0.000001, verbose=1, min_delta=0.001)
callbacks_list = [checkpointer, early, reduce_lr]
keras.backend.clear_session()
model = get_model()
model.fit_generator(train_gen,
steps_per_epoch=nb_train_samples//BATCH_SIZE,
epochs=3,
verbose=1,
callbacks=callbacks_list,
validation_data=valid_gen,
validation_steps=nb_validation_samples//BATCH_SIZE,
max_queue_size=20,
shuffle=True,
workers=4)
for l in model.layers[:-1]:
l.trainable = True
sgd = keras.optimizers.SGD(lr=0.0001, momentum=0.9)
model.compile(optimizer=sgd, loss=losses.categorical_crossentropy, metrics=['acc'])
model.summary()
model.fit_generator(train_gen,
steps_per_epoch=nb_train_samples//BATCH_SIZE,
epochs=200,
verbose=1,
callbacks=callbacks_list,
validation_data=valid_gen,
validation_steps=nb_validation_samples//BATCH_SIZE,
max_queue_size=20,
shuffle=True,
workers=4)
model.save('../../fashion_mnist/inception_resnet_v2_retrain_model.h5')
job:
name: "train-fashion-mnist"
execution:
image: "tensorflow/tensorflow:1.5.0-gpu"
command: "pip install keras --upgrade && python training.py"
completionTime: "10"
resources:
cpus: 1
memory: 10000
gpus: 1
cf sapml job submit -f fashion_mnist.yaml code
cf sapml fs put testing fashion_mnist/
## import libraries
import numpy as np
import pandas as pd
import os
import keras
from keras.preprocessing.image import ImageDataGenerator
from keras.models import load_model
from keras.preprocessing import image
from keras.applications.inception_resnet_v2 import preprocess_input
img_width, img_height = 28, 28
test_data_dir = '../../fashion_mnist/testing/'
datagen = ImageDataGenerator(rescale=1./255, preprocessing_function=preprocess_input)
model = load_model('../../fashion_mnist/inception_resnet_v2_retrain_model.h5')
generator = datagen.flow_from_directory(
test_data_dir,
target_size=(img_width, img_height),
batch_size=128,
seed=27,
class_mode=None, # only data, no labels
shuffle=False) # keep data in same order as labels
img_names = generator.filenames
probs = model.predict_generator(generator, verbose=1)
predictions = np.argmax(probs, axis=1)
df = pd.DataFrame(columns=['Image-Id', 'Category'])
df['id'] = img_names
df['label'] = predictions
df.to_csv('../../fashion_mnist/submit.csv', index=False)
job:
name: "predict-fashion-mnist"
execution:
image: "tensorflow/tensorflow:1.5.0-gpu"
command: "pip install keras --upgrade && python prediction.py"
completionTime: "5"
resources:
cpus: 1
memory: 10000
gpus: 1
cf sapml job submit -f fashion_mnist.yaml code
cf sapml fs get fashion_mnist/submit.csv .
You must be a registered user to add a comment. If you've already registered, sign in. Otherwise, register and sign in.
User | Count |
---|---|
29 | |
17 | |
15 | |
13 | |
11 | |
9 | |
8 | |
8 | |
8 | |
7 |