青空文庫で作者っぽさ判定

"author", "url"natsume, http://www.aozora.gr.jp/index_pages/person148.html#sakuhin_list_1akutagawa, http://www.aozora.gr.jp/index_pages/person879.html#sakuhin_list_1mori, http://www.aozora.gr.jp/index_pages/person129.html#sakuhin_list_1sakaguchi, http://www.aozora.gr.jp/index_pages/person1095.html#sakuhin_list_1yoshikawa, http://www.aozora.gr.jp/index_pages/person1562.html#sakuhin_list_1

# Generating integrated file of csv's of all the pieces for each author.# One csv file is generated and saved for each author.def author_data_integ(auth_target=auth_target): for w in auth_target[1:]: print ("starting: " + w[0]) auth_dir = '{}{}/'.format(aozora_dir, w[0]) csv_dir = '{}{}'.format(auth_dir, "csv/") files = os.listdir(csv_dir) integ_np = np.array([["author", "line"]]) for file in files: if "csv" in file: print (" now at: " + file) file_name = csv_dir + file pds_data = pds.read_csv(file_name, index_col=0) pds_data = pds_data.dropna() np_data = np.array(pds_data.ix[:,[0,2]]) out = [j for j in range(len(np_data)) if '-----------' in str(np_data[j,1])] if not out: out = [1] hyphen_pos = int(out[len(out) - 1]) last_20 = len(np_data) - 20 np_data = np_data[hyphen_pos+1:last_20,:] integ_np = np.vstack((integ_np, np_data)) integ_pds = pds.DataFrame(integ_np[1:,:], columns=integ_np[0,:]) integ_pds.to_csv(auth_dir + w[0] + '_integ.csv', quoting=csv.QUOTE_ALL) print ("finished: " + w[0])

data_integ = np.vstack((np.vstack((np.vstack((natsume, akutagawa)),mori)),sakaguchi))

# Converting each line of texts to encoded array.def load_data(txt, max_length=200): txt_list = [] for l in txt: txt_line = [ord(x) for x in str(l).strip()] # You will get encoded text in array, just like this # [25991, 31456, 12391, 12399, 12394, 12367, 12387, 12390, 23383, 24341, 12391, 12354, 12427, 12290] txt_line = txt_line[:max_length] txt_len = len(txt_line) if txt_len < max_length: txt_line += ([0] * (max_length - txt_len)) txt_list.append((txt_line)) return txt_list

# Creating character-level convolutional neural network model.def create_model(embed_size=128, max_length=200, filter_sizes=(2, 3, 4, 5), filter_num=64): inp = Input(shape=(max_length,)) emb = Embedding(0xffff, embed_size)(inp) emb_ex = Reshape((max_length, embed_size, 1))(emb) convs = [] for filter_size in filter_sizes: conv = Convolution2D(filter_num, filter_size, embed_size, activation="relu")(emb_ex) pool = MaxPooling2D(pool_size=(max_length - filter_size + 1, 1))(conv) convs.append(pool) convs_merged = merge(convs, mode='concat') reshape = Reshape((filter_num * len(filter_sizes),))(convs_merged) fc1 = Dense(64, activation="relu")(reshape) bn1 = BatchNormalization()(fc1) do1 = Dropout(0.5)(bn1) fc2 = Dense(3, activation='sigmoid')(do1) model = Model(input=inp, output=fc2) return model

# Training the model.def train(inputs, targets, batch_size=100, epoch_count=100, max_length=200, model_filepath=aozora_dir + "model.h5", learning_rate=0.001): start = learning_rate stop = learning_rate * 0.01 learning_rates = np.linspace(start, stop, epoch_count) model = create_model(max_length=max_length) optimizer = Adam(lr=learning_rate) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) model.summary() # Load previous weight if exists target = os.path.join('/tmp', 'weights.*.hdf5') files = [(f, os.path.getmtime(f)) for f in glob.glob(target)] if len(files) != 0: latest_saved_model = sorted(files, key=lambda files: files[1])[-1] model.load_weights(latest_saved_model[0]) # Logging file for each epoch csv_logger_file = '/tmp/clcnn_training.log' # Checkpoint model for each epoch checkpoint_filepath = "/tmp/weights.{epoch:02d}-{loss:.2f}-{acc:.2f}-{val_loss:.2f}-{val_acc:.2f}.hdf5" model.fit(inputs, targets, nb_epoch=epoch_count, batch_size=batch_size, verbose=1, validation_split=0.1, shuffle=True, callbacks=[ LearningRateScheduler(lambda epoch: learning_rates[epoch]), CSVLogger(csv_logger_file), ModelCheckpoint(filepath=checkpoint_filepath, verbose=1, save_best_only=True, save_weights_only=False, monitor='val_acc') ]) model.save(model_filepath)

例：nohup python aozora_cnn.py &

cat nohup.out

# coding: utf-8"""This predict how much a sentence is similar to one of the Japanese classical author.Before using this, be sure to generate a model using aozora_cnn.py.Change the values for model_file to your model hdf5 file path, raw_txt to your favorite sentence in Japaneseand target_author to the author you trained the model."""import numpy as npimport pandas as pdsfrom keras.models import load_modelmodel_file = "/tmp/weights.07-0.22-0.92-0.35-0.88.hdf5"raw_txt = "隴西の李徴は博學才穎、天寶の末年、若くして名を虎榜に連ね、ついで江南尉に補せられたが、性、狷介、自ら恃む所頗る厚く、賤吏に甘んずるを潔しとしなかつた。"target_author = ["夏目漱石","芥川龍之介","森鴎外","坂口安吾"]# Encodes the raw_txtdef text_encoding(raw_txt): txt = [ord(x) for x in str(raw_txt).strip()] txt = txt[:200] if len(txt) < 200: txt += ([0] * (200 - len(txt))) return txt# Predictdef predict(comments, model_filepath="model.h5"): model = load_model(model_filepath) ret = model.predict(comments) return retif __name__ == "__main__": txt = text_encoding(raw_txt) predict_result = predict(np.array([txt]), model_filepath=model_file) pds_predict_result = pds.DataFrame(predict_result, columns=target_author)pds_predict_result