SageMaker BlazingText 算法讲解

importsagemaker  

fromsagemaker import get_execution_role  

importboto3  

importjson  


sess = sagemaker.Session()  


role = get_execution_role()  

print(role) # This is the role that SageMaker would use to leverage Amazon Web Services resources (S3, CloudWatch) on your behalf


bucket = '421710401846-sagemaker-us-west-2'# Replace with your own bucket name if needed  

prefix = 'nlp-handson'#Replace with the prefix under which you want to store the data if needed 

!wget https://421710401846-sagemaker-us-west-2.s3-us-west-2.amazonaws.com/nlp-handson/zh-train/wiki_zh_full  

# Install jieba tool

!pip install jieba  

# Download stopwords

!wget https://421710401846-sagemaker-us-west-2.s3-us-west-2.amazonaws.com/nlp-handson/zhstopwords.txt  


importlogging,jieba,os,re  


defget_stopwords():  

basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',level=logging.INFO)  

#加载停用词表  

stopword_set = set()  

with open("zhstopwords.txt",'r',encoding="utf-8") as stopwords:  

for stopword in stopwords:  

add(stopword.strip("\n"))  

return stopword_set

defparse_zh_words(read_file_path,save_file_path):   

file = open(read_file_path,"r",encoding="utf-8")  

#过滤掉英文和数字等特殊字符  

r1 = '[^\u4e00-\u9fa5]'

#写文件  

output = open(save_file_path,"w+",encoding="utf-8")  

content_line = file.readline()  

#获取停用词表  

stopwords = get_stopwords()  

#定义一个字符串变量，表示一篇文章的分词结果  

article_contents = "" 


while content_line:  

content_line = content_line.strip("\n")  

if len(content_line) > 0:  

#去除数字和英文等特殊字符  

zh_content_line = re.sub(r1, '', content_line)  

#使用jieba进行分词  

words = jieba.cut(zh_content_line,cut_all=False)  

……


input_file = './data/wiki_zh_full'

output_file = './data/wiki_zh_corpus'

parse_zh_words(input_file, output_file)  

train_channel = prefix + '/zh-train'


upload_data(path='data/wiki_zh_corpus', bucket=bucket, key_prefix=train_channel)  


s3_train_data = 's3://{}/{}'.format(bucket, train_channel)  

s3_output_location = 's3://{}/{}/zh-output'.format(bucket, prefix)  

container = sagemaker.amazon.amazon_estimator.get_image_uri(region_name, "blazingtext", "latest")

bt_model = sagemaker.estimator.Estimator(container,  

role,   

train_instance_count=1,   

train_instance_type='ml.p3.2xlarge',  

train_use_spot_instances = True,  

train_volume_size = 10,  

train_max_run = 3600,  

train_max_wait = 3600,

input_mode= 'File',  

output_path=s3_output_location,  

sagemaker_session=sess)  

set_hyperparameters(mode= "skipgram", #"batch_skipgram",

epochs=10,  
min_count=5,  
sampling_threshold=0.0001,  
learning_rate=0.05,  
window_size=5,  
vector_dim=300,  
negative_samples=5,  
batch_size=11, 
evaluation=False
subwords=False) 

train_data = sagemaker.session.s3_input(s3_train_data, distribution='FullyReplicated', content_type='text/plain', s3_data_type='S3Prefix')  
data_channels = {'train': train_data} 

fit(inputs=data_channels, logs=True) 

frommanifold importTSNE  

frommatplotlib import pylab  

frompylab import mpl  


tsne = TSNE(perplexity=40, n_components=2, init='pca', n_iter=10000)  

two_d_embeddings = tsne.fit_transform(word_vecs[:num_points])  

labels = index_to_word[:num_points]  


rcParams['font.sans-serif'] = ['SimHei'] # 指定默认字体

rcParams['axes.unicode_minus'] = False # 解决保存图像是负号'-'显示为方块的问题

%matplotlib inline  


defplot(embeddings, labels):  

figure(figsize=(20,20))  

fori, label in enumerate(labels):  

x, y = embeddings[i,:]  

scatter(x, y)  

annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points',ha='right', va='bottom')  

#pylab.savefig('tsne-ch.png')

show()  


plot(two_d_embeddings, labels)  

!pip install gensim  

frommodels importKeyedVectors  


word_vectors = KeyedVectors.load_word2vec_format('vectors.txt', binary=False)  


result1 = word_vectors.most_similar(positive=['女人', '国王'], negative=['男人'])  

print("{}: {:.4f}".format(*result1[0]))  


print(word_vectors.doesnt_match("天才傻瓜 疯子 牛肉".split())) 

!pip install wordcloud  

importpyplot as plt  

fromwordcloud import WordCloud  


...  


#输入一个词找出相似的前100个词

one_corpus = ["美丽"]  

result = word_vectors.most_similar(one_corpus[0],topn=100)  

#将返回的结果转换为字典,便于绘制词云

word_cloud = dict()  

forsim inresult:  

word_cloud[sim[0]] = sim[1]  

#绘制词云

draw_word_cloud(word_cloud)