2단계: 데이터 탐색하기

def load_imdb_sentiment_analysis_dataset(data_path, seed=123):     """Loads the IMDb movie reviews sentiment analysis dataset.      # Arguments         data_path: string, path to the data directory.         seed: int, seed for randomizer.      # Returns         A tuple of training and validation data.         Number of training samples: 25000         Number of test samples: 25000         Number of categories: 2 (0 - negative, 1 - positive)      # References         Mass et al., http://www.aclweb.org/anthology/P11-1015          Download and uncompress archive from:         http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz     """     imdb_data_path = os.path.join(data_path, 'aclImdb')      # Load the training data     train_texts = []     train_labels = []     for category in ['pos', 'neg']:         train_path = os.path.join(imdb_data_path, 'train', category)         for fname in sorted(os.listdir(train_path)):             if fname.endswith('.txt'):                 with open(os.path.join(train_path, fname)) as f:                     train_texts.append(f.read())                 train_labels.append(0 if category == 'neg' else 1)      # Load the validation data.     test_texts = []     test_labels = []     for category in ['pos', 'neg']:         test_path = os.path.join(imdb_data_path, 'test', category)         for fname in sorted(os.listdir(test_path)):             if fname.endswith('.txt'):                 with open(os.path.join(test_path, fname)) as f:                     test_texts.append(f.read())                 test_labels.append(0 if category == 'neg' else 1)      # Shuffle the training data and labels.     random.seed(seed)     random.shuffle(train_texts)     random.seed(seed)     random.shuffle(train_labels)      return ((train_texts, np.array(train_labels)),             (test_texts, np.array(test_labels)))

import numpy as np import matplotlib.pyplot as plt  def get_num_words_per_sample(sample_texts):     """Returns the median number of words per sample given corpus.      # Arguments         sample_texts: list, sample texts.      # Returns         int, median number of words per sample.     """     num_words = [len(s.split()) for s in sample_texts]     return np.median(num_words)  def plot_sample_length_distribution(sample_texts):     """Plots the sample length distribution.      # Arguments         samples_texts: list, sample texts.     """     plt.hist([len(s) for s in sample_texts], 50)     plt.xlabel('Length of a sample')     plt.ylabel('Number of samples')     plt.title('Sample length distribution')     plt.show()